From 92352a1a2391418ce2cce5f96830bf0bc0aab42d Mon Sep 17 00:00:00 2001 From: Andrij Mizyk Date: Tue, 17 Aug 2021 21:28:32 +0000 Subject: [PATCH 01/56] Translated using Weblate (Ukrainian) Currently translated at 60.0% (874 of 1456 strings) --- plinth/locale/uk/LC_MESSAGES/django.po | 107 ++++++++++++++++--------- 1 file changed, 71 insertions(+), 36 deletions(-) diff --git a/plinth/locale/uk/LC_MESSAGES/django.po b/plinth/locale/uk/LC_MESSAGES/django.po index 06aeb9c41..f50c92ae2 100644 --- a/plinth/locale/uk/LC_MESSAGES/django.po +++ b/plinth/locale/uk/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2021-08-16 18:38-0400\n" -"PO-Revision-Date: 2021-08-10 19:32+0000\n" +"PO-Revision-Date: 2021-08-17 21:29+0000\n" "Last-Translator: Andrij Mizyk \n" "Language-Team: Ukrainian \n" @@ -117,7 +117,7 @@ msgid "Access URL {url}" msgstr "Доступ до URL {url}" #: plinth/modules/avahi/__init__.py:35 -#, fuzzy, python-brace-format +#, python-brace-format msgid "" "Service discovery allows other devices on the network to discover your " "{box_name} and services running on it. It also allows {box_name} to " @@ -126,11 +126,11 @@ msgid "" "disabled to improve security especially when connecting to a hostile local " "network." msgstr "" -"Виявлення служб дозволяє іншим пристроям в мережі знаходити ваш {box_name} " -"та запущені на ньому служби. Це дозволяє {box_name} знаходити інші пристрої " -"та служби запущені у вашій локальній мережі. Знаходження служб не головне і " -"працює лише у внутрішніх мережах. Це можна виключити, для підвищення " -"безпеки, особливо при підключенні до не довіреної мережі." +"Виявлення сервісів дозволяє іншим пристроям у мережі знаходити ваш {box_name}" +" та запущені на ньому сервіси. Це дозволяє {box_name} знаходити інші " +"пристрої та сервіси запущені у вашій локальній мережі. Знаходження сервісів " +"не головне і працює лише у внутрішніх мережах. Це можна вимкнути для " +"підвищення безпеки, особливо при підключенні до не довіреної мережі." #: plinth/modules/avahi/__init__.py:59 msgid "Service Discovery" @@ -397,6 +397,8 @@ msgid "" "Connection refused - make sure you provided correct credentials and the " "server is running." msgstr "" +"У зʼєднані відмовлено - переконайтеся, що ви надали правильні облікові дані " +"і чи сервер працює." #: plinth/modules/backups/repository.py:41 msgid "Connection refused" @@ -417,6 +419,7 @@ msgstr "Доступ SSH відхилено" #: plinth/modules/backups/repository.py:64 msgid "Repository path is neither empty nor is an existing backups repository." msgstr "" +"Шлях репозиторію непорожній і не є наявним репозиторієм резервних копій." #: plinth/modules/backups/repository.py:143 msgid "Existing repository is not encrypted." @@ -471,6 +474,9 @@ msgid "" "To restore a backup on a new %(box_name)s you need the ssh credentials and, " "if chosen, the encryption passphrase." msgstr "" +"Облікові дані для цього репозиторію зберігаються на вашому %(box_name)s. " +"
Щоб відновити резервну копію на новому %(box_name)s вам потрібні " +"облікові дані ssh та, якщо вибрано, парольна фраза для шифрування." #: plinth/modules/backups/templates/backups_add_remote_repository.html:28 msgid "Create Location" @@ -552,6 +558,8 @@ msgid "" "The remote repository will not be deleted. This just removes the repository " "from the listing on the backup page, you can add it again later on." msgstr "" +"Віддалений репозиторій не видалиться. Це лише вилучить репозиторій зі списку " +"на сторінці резервування, ви можете додати його пізніше." #: plinth/modules/backups/templates/backups_repository_remove.html:31 msgid "Remove Location" @@ -572,12 +580,19 @@ msgid "" " backup file.\n" " " msgstr "" +"\n" +" Вивантажити файл резервної копії від іншого %(box_name)s, щоб " +"відновити\n" +" його вміст. Можете обрати застосунки, які бажаєте відновити, після " +"вивантажування\n" +" файлу резервної копії.\n" +" " #: plinth/modules/backups/templates/backups_upload.html:27 #: plinth/modules/help/templates/statuslog.html:23 #: plinth/modules/networks/templates/connection_show.html:23 msgid "Caution:" -msgstr "" +msgstr "Обережно:" #: plinth/modules/backups/templates/backups_upload.html:28 #, python-format @@ -585,6 +600,8 @@ msgid "" "You have %(max_filesize)s available to restore a backup. Exceeding this " "limit can leave your %(box_name)s unusable." msgstr "" +"У вас доступно %(max_filesize)s для відновлення резервної копії. Перевищення " +"цього обмеження може зробити ваш %(box_name)s не придатним для використання." #: plinth/modules/backups/templates/backups_upload.html:41 msgid "Upload file" @@ -596,6 +613,8 @@ msgid "" "Could not reach SSH host %(hostname)s. Please verify that the host is up and " "accepting connections." msgstr "" +"Не вдалося звʼязатися з власником SSH %(hostname)s. Перевірте чи компʼютер " +"працює і приймає зʼєднання." #: plinth/modules/backups/templates/verify_ssh_hostkey.html:28 #, python-format @@ -603,6 +622,8 @@ msgid "" "The authenticity of SSH host %(hostname)s could not be established. The host " "advertises the following SSH public keys. Please verify any one of them." msgstr "" +"Не вдалося встановити автентифікацію власника SSH %(hostname)s. Компʼютер " +"повідомляє про такі публічні ключі SSH. Перевірте будь-який з них." #: plinth/modules/backups/templates/verify_ssh_hostkey.html:40 msgid "How to verify?" @@ -614,10 +635,13 @@ msgid "" "one of the provided options. You can also use dsa, ecdsa, ed25519 etc. " "instead of rsa, by choosing the corresponding file." msgstr "" +"Запустіть наступні команди на машині, що розміщує SSH. Вивід має відповідати " +"одному з наведених параметрів. Ви також можете використовувати dsa, ecdsa, " +"ed25519 і т.п. замість rsa, обравши відповідний файл." #: plinth/modules/backups/templates/verify_ssh_hostkey.html:60 msgid "Verify Host" -msgstr "" +msgstr "Перевірити власника" #: plinth/modules/backups/views.py:55 msgid "Backup schedule updated." @@ -673,19 +697,19 @@ msgstr "Додано новий віддалений SSH-репозиторій. #: plinth/modules/backups/views.py:366 msgid "Verify SSH hostkey" -msgstr "" +msgstr "Перевірити ключ власника SSH" #: plinth/modules/backups/views.py:392 msgid "SSH host already verified." -msgstr "" +msgstr "Власника SSH вже перевірено." #: plinth/modules/backups/views.py:402 msgid "SSH host verified." -msgstr "" +msgstr "Власника SSH перевірено." #: plinth/modules/backups/views.py:417 msgid "SSH host public key could not be verified." -msgstr "" +msgstr "Публічний ключ власника SSH не можливо перевірити." #: plinth/modules/backups/views.py:419 msgid "Authentication to remote server failed." @@ -722,6 +746,11 @@ msgid "" "audio, video and PDF documents can be previewed in the browser. Shared files " "can be set to expire after a time period." msgstr "" +"bepasty — це вебзастосунок, що дозволяє відвантажувати і ділитися великими " +"файлами. Текст і фрагменти коду також можна вставляти і ширити. Текст, " +"картинки, звук, відеозаписи та документи PDF можна попередньо переглянути в " +"оглядачі. Поширювані файли можна позначати застарілими після певного періоду " +"часу." #: plinth/modules/bepasty/__init__.py:28 msgid "" @@ -730,6 +759,9 @@ msgid "" "password, you can share it with the users who should have the associated " "permissions." msgstr "" +"bepasty не використовує імена користувачів для входу. Використовує лише " +"паролі. Для кожного пароля можна вибирати набір дозволів. Створивши пароль, " +"ви зможете ділитися ним із користувачами, які мають відповідні дозволи." #: plinth/modules/bepasty/__init__.py:32 msgid "" @@ -738,18 +770,21 @@ msgid "" "revoke access for a single person or group, by removing their password from " "the list." msgstr "" +"Можна створити декілька паролів з однаковим набором привілеїв і роздати їх " +"різним людям або групам. Це дозволить скасувати доступ до певної особи або " +"групи видаливши її пароль зі списку." #: plinth/modules/bepasty/__init__.py:41 plinth/modules/bepasty/__init__.py:50 msgid "Read a file, if a web link to the file is available" -msgstr "" +msgstr "Прочитати файл, якщо вебпосилання на файл доступне" #: plinth/modules/bepasty/__init__.py:42 msgid "Create or upload files" -msgstr "" +msgstr "Створити або відвантажити файли" #: plinth/modules/bepasty/__init__.py:43 msgid "List all files and their web links" -msgstr "" +msgstr "Показати всі файли і їхні вебпосилання" #: plinth/modules/bepasty/__init__.py:44 msgid "Delete files" @@ -757,42 +792,42 @@ msgstr "Видалити файли" #: plinth/modules/bepasty/__init__.py:45 msgid "Administer files: lock/unlock files" -msgstr "" +msgstr "Адміністрування файлів: блокувати/розблокувати файли" #: plinth/modules/bepasty/__init__.py:49 msgid "None, password is always required" -msgstr "" +msgstr "Нічого, пароль обовʼязковий" #: plinth/modules/bepasty/__init__.py:51 msgid "List and read all files" -msgstr "" +msgstr "Показати і зчитати всі файли" #: plinth/modules/bepasty/__init__.py:64 plinth/modules/bepasty/manifest.py:6 msgid "bepasty" -msgstr "" +msgstr "bepasty" #: plinth/modules/bepasty/__init__.py:66 msgid "File & Snippet Sharing" -msgstr "" +msgstr "Обмін файлами і фрагментами коду" #: plinth/modules/bepasty/forms.py:17 msgid "Public Access (default permissions)" -msgstr "" +msgstr "Публічний доступ (типові дозволи)" #: plinth/modules/bepasty/forms.py:18 msgid "Permissions for anonymous users, who have not provided a password." -msgstr "" +msgstr "Дозволи для анонімних користувачів, що не надали пароль." #: plinth/modules/bepasty/forms.py:27 #: plinth/modules/bepasty/templates/bepasty.html:30 #: plinth/modules/users/forms.py:103 plinth/modules/users/forms.py:227 msgid "Permissions" -msgstr "" +msgstr "Дозволи" #: plinth/modules/bepasty/forms.py:29 msgid "" "Users that log in with this password will have the selected permissions." -msgstr "" +msgstr "Користувачі, що ввійдуть із цим паролем матимуть вибрані дозволи." #: plinth/modules/bepasty/forms.py:33 #: plinth/modules/bepasty/templates/bepasty.html:31 @@ -801,7 +836,7 @@ msgstr "Коментар" #: plinth/modules/bepasty/forms.py:34 msgid "Any comment to help you remember the purpose of this password." -msgstr "" +msgstr "Будь-який коментар, що допоможе запамʼятати призначення цього пароля." #: plinth/modules/bepasty/templates/bepasty.html:12 msgid "Manage Passwords" @@ -814,7 +849,7 @@ msgstr "Додати пароль" #: plinth/modules/bepasty/templates/bepasty.html:23 msgid "No passwords currently configured." -msgstr "" +msgstr "Не налаштовано жодного пароля." #: plinth/modules/bepasty/templates/bepasty.html:29 #: plinth/modules/dynamicdns/forms.py:106 plinth/modules/networks/forms.py:213 @@ -824,7 +859,7 @@ msgstr "Пароль" #: plinth/modules/bepasty/views.py:23 msgid "admin" -msgstr "" +msgstr "адміністратор" #: plinth/modules/bepasty/views.py:24 msgid "editor" @@ -832,11 +867,11 @@ msgstr "редактор" #: plinth/modules/bepasty/views.py:25 msgid "viewer" -msgstr "" +msgstr "переглядач" #: plinth/modules/bepasty/views.py:50 msgid "Read" -msgstr "" +msgstr "Читати" #: plinth/modules/bepasty/views.py:51 msgid "Create" @@ -844,7 +879,7 @@ msgstr "Створити" #: plinth/modules/bepasty/views.py:52 msgid "List" -msgstr "" +msgstr "Показати" #: plinth/modules/bepasty/views.py:53 #: plinth/modules/letsencrypt/templates/letsencrypt.html:86 @@ -855,23 +890,23 @@ msgstr "" #: plinth/modules/wireguard/templates/wireguard_show_client.html:77 #: plinth/modules/wireguard/templates/wireguard_show_server.html:78 msgid "Delete" -msgstr "" +msgstr "Видалити" #: plinth/modules/bepasty/views.py:54 msgid "Admin" -msgstr "" +msgstr "Адміністратор" #: plinth/modules/bepasty/views.py:91 plinth/modules/searx/views.py:38 #: plinth/modules/searx/views.py:49 plinth/modules/tor/views.py:130 #: plinth/modules/tor/views.py:157 plinth/modules/zoph/views.py:69 msgid "Configuration updated." -msgstr "" +msgstr "Налаштування оновлено." #: plinth/modules/bepasty/views.py:94 plinth/modules/gitweb/views.py:117 #: plinth/modules/searx/views.py:41 plinth/modules/searx/views.py:52 #: plinth/modules/tor/views.py:159 plinth/modules/zoph/views.py:72 msgid "An error occurred during configuration." -msgstr "" +msgstr "Під час налаштування відбулася помилка." #: plinth/modules/bepasty/views.py:105 msgid "Password added." @@ -2573,7 +2608,7 @@ msgstr "" #: plinth/modules/infinoted/__init__.py:26 msgid "infinoted is a server for Gobby, a collaborative text editor." -msgstr "" +msgstr "infinoted — це сервер для Gobby, колективного редактора тексту." #: plinth/modules/infinoted/__init__.py:28 #, python-brace-format From 3def7706db48068fef555e4eea8ff5834fe127d8 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Mon, 7 Jun 2021 21:33:35 +0000 Subject: [PATCH 02/56] diagnostics: Allow underscores (_) in app names Reviewed-by: Sunil Mohan Adapa --- plinth/modules/diagnostics/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plinth/modules/diagnostics/urls.py b/plinth/modules/diagnostics/urls.py index 2c6c692ba..63b843670 100644 --- a/plinth/modules/diagnostics/urls.py +++ b/plinth/modules/diagnostics/urls.py @@ -9,6 +9,6 @@ from . import views urlpatterns = [ url(r'^sys/diagnostics/$', views.index, name='index'), - url(r'^sys/diagnostics/(?P[1-9a-z\-]+)/$', views.diagnose_app, + url(r'^sys/diagnostics/(?P[1-9a-z\-_]+)/$', views.diagnose_app, name='app'), ] From 585433d427552e70f2ccfea2ab520a13ebddba49 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Tue, 17 Aug 2021 18:51:49 -0700 Subject: [PATCH 03/56] doc/dev: Using mocking instead of importing external modules [sunil: Expand the full list of modules] Signed-off-by: Sunil Mohan Adapa Reviewed-by: Sunil Mohan Adapa --- doc/dev/conf.py | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/doc/dev/conf.py b/doc/dev/conf.py index 80d680a1b..937d93efd 100644 --- a/doc/dev/conf.py +++ b/doc/dev/conf.py @@ -16,24 +16,8 @@ list see the documentation: http://www.sphinx-doc.org/en/master/config import os import sys -import django -import django.conf - sys.path.insert(0, os.path.abspath('../../')) -# -- Django setup ------------------------------------------------------------ - -# Ensure that Django models can be imported by Sphinx -import plinth.settings # noqa pylint: disable=E402 isort:skip - -kwargs = {} -for setting in dir(plinth.settings): - if setting.isupper(): - kwargs[setting] = getattr(plinth.settings, setting) - -django.conf.settings.configure(**kwargs) -django.setup(set_prefix=True) - # -- Project information ----------------------------------------------------- # pylint: disable=invalid-name @@ -209,4 +193,29 @@ autodoc_default_options = { 'special-members': '__init__', } +# Mock importing packages +autodoc_mock_imports = [ + 'apt', + 'augeas', + 'axes', + 'bootstrapform', + 'captcha', + 'cherrypy', + 'configobj', + 'dbus', + 'django', + 'gi', + 'markupsafe', + 'OpenSSL', + 'pam', + 'paramiko', + 'psutil', + 'pytest', + 'requests', + 'ruamel', + 'setuptools', + 'stronghold', + 'yaml', +] + html_favicon = './_static/favicon.ico' From 673ed475b7ca688c04d64b7282940e7a7bfbd910 Mon Sep 17 00:00:00 2001 From: Sunil Mohan Adapa Date: Tue, 17 Aug 2021 19:25:06 -0700 Subject: [PATCH 04/56] doc/dev: Drop seemingly irrelevant automodule reference Signed-off-by: Sunil Mohan Adapa --- doc/dev/reference/index.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/dev/reference/index.rst b/doc/dev/reference/index.rst index 1a16ce3e5..bf40769aa 100644 --- a/doc/dev/reference/index.rst +++ b/doc/dev/reference/index.rst @@ -20,5 +20,3 @@ and are updated when the API is updated. views forms notifications - -.. automodule:: plinth.modules.ttrss From aaa6342f936c1a723ec339359c55b92700a3739a Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Sun, 28 Mar 2021 23:40:18 +0000 Subject: [PATCH 05/56] email: Basic app to manage an email server - Install postfix and manage the service - Import ugettext - Dummy forms and views - .version (integer) is required [sunil: Disable the app until remaining issues are worked out] Signed-off-by: Sunil Mohan Adapa --- actions/email_server | 0 plinth/modules/email_server/__init__.py | 88 +++++++++++++++++++ .../etc/plinth/modules-enabled/email_server | 1 + plinth/modules/email_server/forms.py | 6 ++ plinth/modules/email_server/manifest.py | 52 +++++++++++ plinth/modules/email_server/tests/__init__.py | 0 plinth/modules/email_server/urls.py | 8 ++ plinth/modules/email_server/views.py | 15 ++++ 8 files changed, 170 insertions(+) create mode 100644 actions/email_server create mode 100644 plinth/modules/email_server/__init__.py create mode 100644 plinth/modules/email_server/data/etc/plinth/modules-enabled/email_server create mode 100644 plinth/modules/email_server/forms.py create mode 100644 plinth/modules/email_server/manifest.py create mode 100644 plinth/modules/email_server/tests/__init__.py create mode 100644 plinth/modules/email_server/urls.py create mode 100644 plinth/modules/email_server/views.py diff --git a/actions/email_server b/actions/email_server new file mode 100644 index 000000000..e69de29bb diff --git a/plinth/modules/email_server/__init__.py b/plinth/modules/email_server/__init__.py new file mode 100644 index 000000000..65211a9c0 --- /dev/null +++ b/plinth/modules/email_server/__init__.py @@ -0,0 +1,88 @@ +"""FreedomBox email server app""" +# SPDX-License-Identifier: AGPL-3.0-or-later + +from django.urls import reverse_lazy +from django.utils.translation import ugettext_lazy as _ + +import plinth.app +import plinth.daemon +import plinth.frontpage +import plinth.menu +from plinth.modules.firewall.components import Firewall + +from . import manifest + +version = 1 +managed_packages = ['postfix', 'dovecot-pop3d', 'dovecot-imapd', + 'dovecot-lmtpd', 'dovecot-ldap', 'dovecot-managesieved'] +managed_services = ['postfix', 'dovecot'] +app = None + + +class EmailServerApp(plinth.app.App): + """FreedomBox email server app""" + app_id = 'email_server' + + def __init__(self): + """The app's constructor""" + super().__init__() + + info = plinth.app.Info( + app_id=self.app_id, + version=version, + name=_('Email Server'), + short_description=_('An email server for FreedomBox'), + manual_page='EmailServer', + clients=manifest.clients, + donation_url='https://freedomboxfoundation.org/donate/' + ) + self.add(info) + + menu_item = plinth.menu.Menu( + 'menu_' + self.app_id, # unique id + info.name, # app display name + info.short_description, # app description + 'roundcube', # icon name in `static/theme/icons/` + 'email_server:index', # view name + parent_url_name='apps' + ) + self.add(menu_item) + + shortcut = plinth.frontpage.Shortcut( + 'shortcut_' + self.app_id, + name=info.name, + short_description=info.short_description, + icon='roundcube', + configure_url=reverse_lazy('email_server'), + clients=manifest.clients, + login_required=True + ) + self.add(shortcut) + + postfix_ports = [] + dovecot_ports = [] + all_firewalld_ports = [] + for port in (25, 465, 587): + postfix_ports.extend([(port, 'tcp4'), (port, 'tcp6')]) + for port in (993, 995): + dovecot_ports.extend([(port, 'tcp4'), (port, 'tcp6')]) + all_firewalld_ports.extend(['smtp', 'smtps', 'smtp-submission']) + all_firewalld_ports.extend(['pop3s', 'imaps']) + + # Manage daemons + postfixd = plinth.daemon.Daemon('daemon-postfix', 'postfix', + listen_ports=postfix_ports) + dovecotd = plinth.daemon.Daemon('daemon-dovecot', 'dovecot', + listen_ports=dovecot_ports) + self.add(postfixd) + self.add(dovecotd) + + # Ports + firewall = Firewall('firewall-email', info.name, + ports=all_firewalld_ports, is_external=True) + self.add(firewall) + + +def setup(helper, old_version=None): + """Installs and configures module""" + helper.install(managed_packages) diff --git a/plinth/modules/email_server/data/etc/plinth/modules-enabled/email_server b/plinth/modules/email_server/data/etc/plinth/modules-enabled/email_server new file mode 100644 index 000000000..fb87084c0 --- /dev/null +++ b/plinth/modules/email_server/data/etc/plinth/modules-enabled/email_server @@ -0,0 +1 @@ +#plinth.modules.email_server diff --git a/plinth/modules/email_server/forms.py b/plinth/modules/email_server/forms.py new file mode 100644 index 000000000..4f6ecdac7 --- /dev/null +++ b/plinth/modules/email_server/forms.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +from django import forms + + +class EmailServerForm(forms.Form): + pass diff --git a/plinth/modules/email_server/manifest.py b/plinth/modules/email_server/manifest.py new file mode 100644 index 000000000..3a6bdae77 --- /dev/null +++ b/plinth/modules/email_server/manifest.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +from django.utils.translation import ugettext_lazy as _ +from plinth.clients import store_url + +clients = [{ + 'name': _('Roundcube'), + 'platforms': [{ + 'type': 'web', + 'url': '/plinth/apps/roundcube' + }] +}, { + 'name': _('Thunderbird'), + 'platforms': [{ + 'type': 'download', + 'os': 'gnu-linux', + 'url': 'https://www.thunderbird.net/en-US/' + }, { + 'type': 'download', + 'os': 'macos', + 'url': 'https://www.thunderbird.net/en-US/' + }, { + 'type': 'download', + 'os': 'windows', + 'url': 'https://www.thunderbird.net/en-US/' + }] +}, { + 'name': _('K-9 Mail'), + 'platforms': [{ + 'type': 'store', + 'os': 'android', + 'store_name': 'f-droid', + 'url': store_url('f-droid', 'com.fsck.k9') + }, { + 'type': 'store', + 'os': 'android', + 'store_name': 'google-play', + 'url': store_url('google-play', 'com.fsck.k9') + }] +}, { + 'name': _('FairEmail'), + 'platforms': [{ + 'type': 'store', + 'os': 'android', + 'store_name': 'f-droid', + 'url': store_url('f-droid', 'eu.faircode.email') + }, { + 'type': 'store', + 'os': 'android', + 'store_name': 'google-play', + 'url': store_url('google-play', 'eu.faircode.email') + }] +}] diff --git a/plinth/modules/email_server/tests/__init__.py b/plinth/modules/email_server/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/plinth/modules/email_server/urls.py b/plinth/modules/email_server/urls.py new file mode 100644 index 000000000..61d5844af --- /dev/null +++ b/plinth/modules/email_server/urls.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +from django.urls import path +from . import views + + +urlpatterns = [ + path('apps/email_server/', views.EmailServerView.as_view(), name='index') +] diff --git a/plinth/modules/email_server/views.py b/plinth/modules/email_server/views.py new file mode 100644 index 000000000..615142512 --- /dev/null +++ b/plinth/modules/email_server/views.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +import plinth +from . import forms + + +class EmailServerView(plinth.views.AppView): + """Server configuration page""" + app_id = 'email_server' + form_class = forms.EmailServerForm + + def form_valid(self, form): + # old_settings = form.initial + # new_status = form.cleaned_data + # plinth.actions.superuser_run('email_server', ['--help']) + return super().form_valid(form) From cde0b470642c7c28e28379342d816bb214f6377a Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Wed, 16 Jun 2021 04:04:31 +0000 Subject: [PATCH 06/56] email: Enable LDAP by calling postconf in a thread-safe way - Implemented `email_server ipc postconf_set_many_v1` - Implemented `lock.Mutex` (fcntl.lockf and threading.Lock based mutex) - FIXME: Lock file permissions - Implemented `postconf` (thread-safe postconf operations) - Started using service orientation --- actions/email_server | 67 ++++++++++++++++ plinth/modules/email_server/__init__.py | 9 +++ plinth/modules/email_server/audit/__init__.py | 5 ++ plinth/modules/email_server/audit/domain.py | 14 ++++ plinth/modules/email_server/audit/ldap.py | 25 ++++++ plinth/modules/email_server/audit/models.py | 32 ++++++++ plinth/modules/email_server/lock.py | 39 +++++++++ plinth/modules/email_server/postconf.py | 80 +++++++++++++++++++ 8 files changed, 271 insertions(+) mode change 100644 => 100755 actions/email_server create mode 100644 plinth/modules/email_server/audit/__init__.py create mode 100644 plinth/modules/email_server/audit/domain.py create mode 100644 plinth/modules/email_server/audit/ldap.py create mode 100644 plinth/modules/email_server/audit/models.py create mode 100644 plinth/modules/email_server/lock.py create mode 100644 plinth/modules/email_server/postconf.py diff --git a/actions/email_server b/actions/email_server old mode 100644 new mode 100755 index e69de29bb..8d83ad2c0 --- a/actions/email_server +++ b/actions/email_server @@ -0,0 +1,67 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: AGPL-3.0-or-later + +import logging +import os +import signal +import sys + +import plinth.modules.email_server.postconf as postconf + +logger = logging.getLogger(__name__) + +EXIT_SYNTAX = 10 +EXIT_TIMEOUT = 20 + + +def main(): + if len(sys.argv) != 3: + sys.exit(EXIT_SYNTAX) + if sys.argv[1] != 'ipc': + sys.exit(EXIT_SYNTAX) + + function_name = 'ipc_' + sys.argv[2] + globals()[function_name]() + + +def ipc_postconf_set_many_v1(): + """Set postconf values""" + if os.getuid() != 0: + logger.warning('Run as root?') + new_config = _postconf_parse_stdin() + with postconf.postconf_mutex.lock_all(): + for key, value in new_config.items(): + postconf.set_no_lock_assuming_root(key, value) + + +def _postconf_parse_stdin(): + new_config = {} + # Set timeout handler + signal.signal(signal.SIGALRM, _timeout_handler) + while True: + key = _timed_input('Key: ') + if not key: break + postconf.validate_key(key) + value = _timed_input('Value: ') + postconf.validate_value(value) + new_config[key] = value + return new_config + + +def _timed_input(prompt): + # Set timeout + signal.alarm(3) + result = input(prompt) + # Disable timeout + signal.alarm(0) + return result + + +def _timeout_handler(signum, frame): + # https://stackoverflow.com/a/1336751 + logger.critical('[Time out]') + sys.exit(EXIT_TIMEOUT) + + +if __name__ == '__main__': + main() diff --git a/plinth/modules/email_server/__init__.py b/plinth/modules/email_server/__init__.py index 65211a9c0..e8e78d0a2 100644 --- a/plinth/modules/email_server/__init__.py +++ b/plinth/modules/email_server/__init__.py @@ -10,6 +10,7 @@ import plinth.frontpage import plinth.menu from plinth.modules.firewall.components import Firewall +from . import audit from . import manifest version = 1 @@ -82,7 +83,15 @@ class EmailServerApp(plinth.app.App): ports=all_firewalld_ports, is_external=True) self.add(firewall) + def diagnose(self): + """Run diagnostics and return the results""" + results = super().diagnose() + results.append(audit.domain.get().summarize()) + results.append(audit.ldap.get().summarize()) + return results + def setup(helper, old_version=None): """Installs and configures module""" helper.install(managed_packages) + helper.call('post', audit.ldap.repair) diff --git a/plinth/modules/email_server/audit/__init__.py b/plinth/modules/email_server/audit/__init__.py new file mode 100644 index 000000000..d82a9cd89 --- /dev/null +++ b/plinth/modules/email_server/audit/__init__.py @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +from . import ldap +from . import domain + +__all__ = ['ldap', 'domain'] diff --git a/plinth/modules/email_server/audit/domain.py b/plinth/modules/email_server/audit/domain.py new file mode 100644 index 000000000..65815a132 --- /dev/null +++ b/plinth/modules/email_server/audit/domain.py @@ -0,0 +1,14 @@ +"""The domain audit resource""" +# SPDX-License-Identifier: AGPL-3.0-or-later + +from . import models + + +def get(): + # Stub + return models.Result('Email domains') + + +def repair(): + # Stub + raise RuntimeError() diff --git a/plinth/modules/email_server/audit/ldap.py b/plinth/modules/email_server/audit/ldap.py new file mode 100644 index 000000000..4da939640 --- /dev/null +++ b/plinth/modules/email_server/audit/ldap.py @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +import plinth.modules.email_server.postconf as postconf +from . import models + +default_config = { + 'smtpd_sasl_auth_enable': 'yes', + 'smtpd_sasl_type': 'dovecot', + 'smtpd_sasl_path': 'private/auth' +} + + +# GET /audit/domain +def get(): + """Compare current values with the default. Generate an audit report""" + results = models.Result('LDAP for user accounts') + current_config = postconf.get_many(list(default_config.keys())) + for key, value in default_config.items(): + if current_config[key] != value: + results.fails.append('{} should equal {}'.format(key, value)) + return results + + +# POST /audit/domain/repair +def repair(): + postconf.set_many(default_config) diff --git a/plinth/modules/email_server/audit/models.py b/plinth/modules/email_server/audit/models.py new file mode 100644 index 000000000..ec91a9280 --- /dev/null +++ b/plinth/modules/email_server/audit/models.py @@ -0,0 +1,32 @@ +"""Audit models""" +# SPDX-License-Identifier: AGPL-3.0-or-later +import logging + +logger = logging.getLogger(__name__) + + +class Result: + def __init__(self, title): + self.title = title + self.fails = [] + self.errors = [] + + def summarize(self, log=True): + """Return a 2-element list for the diagnose function in AppView""" + if log: + self.write_logs() + + if self.errors: + return [self.title, 'error'] + elif self.fails: + return [self.title, 'failed'] + else: + return [self.title, 'passed'] + + def write_logs(self): + """Log errors and failures""" + logger.debug('Ran audit: ' + self.title) + for message in self.errors: + logger.critical(message) + for message in self.fails: + logger.error(message) diff --git a/plinth/modules/email_server/lock.py b/plinth/modules/email_server/lock.py new file mode 100644 index 000000000..bdf138ba7 --- /dev/null +++ b/plinth/modules/email_server/lock.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +import contextlib +import fcntl +import os +import threading + + +class Mutex: + """File and pthread lock based resource mutex""" + + def __init__(self, lock_file): + self.thread_mutex = threading.Lock() + self.lock_path = '/var/lock/' + lock_file + + @contextlib.contextmanager + def lock_threads_only(self): + """Acquire the thread lock but not the file lock""" + self.thread_mutex.acquire(timeout=5) + try: + yield + finally: + self.thread_mutex.release() + + @contextlib.contextmanager + def lock_all(self): + """Acquire both the thread lock and the file lock""" + with self.lock_threads_only(): + fd = open(self.lock_path, 'wb') + # FIXME: Who can lock? + try: + os.fchmod(fd.fileno(), 0o666) # rw-rw-rw- + except OSError: + pass + fcntl.lockf(fd, fcntl.LOCK_EX) + try: + yield + finally: + fcntl.lockf(fd, fcntl.LOCK_UN) + fd.close() diff --git a/plinth/modules/email_server/postconf.py b/plinth/modules/email_server/postconf.py new file mode 100644 index 000000000..b4dd8fee7 --- /dev/null +++ b/plinth/modules/email_server/postconf.py @@ -0,0 +1,80 @@ +"""Postconf wrapper providing thread-safe operations""" +# SPDX-License-Identifier: AGPL-3.0-or-later + +import re +import subprocess +import plinth.actions +from .lock import Mutex + +postconf_mutex = Mutex('plinth-email-postconf.lock') + + +def get_many(key_list): + """Acquire resource lock. Get the list of postconf values as specified. + Return a key-value map""" + result = {} + with postconf_mutex.lock_all(): + for key in key_list: + result[key] = get_no_lock(key) + return result + + +def set_many(kv_map): + """Acquire resource lock. Set the list of postconf values as specified""" + # Encode email_server ipc input + lines = [] + for key, value in kv_map.items(): + validate_key(key) + validate_value(value) + lines.append(key) + lines.append(value) + lines.append('\n') + stdin = '\n'.join(lines).encode('utf-8') + + # Run action script as root + args = ['ipc', 'postconf_set_many_v1'] + with postconf_mutex.lock_threads_only(): + # The action script will take care of file locking + plinth.actions.superuser_run('email_server', args, input=stdin) + + +def get_no_lock(key): + """Get postconf value (no locking)""" + validate_key(key) + result = _run(['/sbin/postconf', key]) + match = key + ' = ' + if not result.startswith(match): + raise KeyError(key) + return result[len(match):].strip() + + +def set_no_lock_assuming_root(key, value): + """Set postconf value (assuming root and no locking)""" + validate_key(key) + validate_value(value) + _run(['/sbin/postconf', '{}={}'.format(key, value)]) + + +def _run(args): + """Run process. Capture and return standard output as a string. Raise a + RuntimeError on non-zero exit codes""" + try: + result = subprocess.run(args, check=True, capture_output=True) + return result.stdout.decode('utf-8') + except subprocess.SubprocessError as subprocess_error: + raise RuntimeError('Subprocess failed') from subprocess_error + except UnicodeDecodeError as unicode_error: + raise RuntimeError('Unicode decoding failed') from unicode_error + + +def validate_key(key): + """Validate postconf key format. Raises ValueError""" + if not re.match('^[a-zA-Z][a-zA-Z0-9_]*$', key): + raise ValueError('Invalid postconf key format') + + +def validate_value(value): + """Validate postconf value format. Raises ValueError""" + for c in value: + if ord(c) < 32: + raise ValueError('Value contains control characters') From fdc6f2390815dae7c24ecb2d17e654750629ca61 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Thu, 17 Jun 2021 00:57:46 +0000 Subject: [PATCH 07/56] email: Implement `email_server ipc set_sasl` and `set_submission` - Rewrote action script to eliminate stdin communication - Changed return type of audit.*.get() - An audit can return multiple lines of diagnostics - Move recommended endpoint URLs into function docstrings --- actions/email_server | 55 +++++----------- plinth/modules/email_server/__init__.py | 6 +- plinth/modules/email_server/audit/domain.py | 2 +- plinth/modules/email_server/audit/ldap.py | 45 +++++++++++-- plinth/modules/email_server/postconf.py | 71 +++++++++++++++------ 5 files changed, 112 insertions(+), 67 deletions(-) diff --git a/actions/email_server b/actions/email_server index 8d83ad2c0..4512367f5 100755 --- a/actions/email_server +++ b/actions/email_server @@ -3,15 +3,23 @@ import logging import os -import signal import sys -import plinth.modules.email_server.postconf as postconf +import plinth.modules.email_server.audit as audit logger = logging.getLogger(__name__) EXIT_SYNTAX = 10 -EXIT_TIMEOUT = 20 +EXIT_PERM = 20 + + +def reserved_for_root(fun): + def wrapped(*args, **kwargs): + if os.getuid() != 0: + logger.critical('This action is reserved for root') + sys.exit(EXIT_PERM) + return fun(*args, **kwargs) + return wrapped def main(): @@ -24,43 +32,14 @@ def main(): globals()[function_name]() -def ipc_postconf_set_many_v1(): - """Set postconf values""" - if os.getuid() != 0: - logger.warning('Run as root?') - new_config = _postconf_parse_stdin() - with postconf.postconf_mutex.lock_all(): - for key, value in new_config.items(): - postconf.set_no_lock_assuming_root(key, value) +@reserved_for_root +def ipc_set_sasl(): + audit.ldap.action_set_sasl() -def _postconf_parse_stdin(): - new_config = {} - # Set timeout handler - signal.signal(signal.SIGALRM, _timeout_handler) - while True: - key = _timed_input('Key: ') - if not key: break - postconf.validate_key(key) - value = _timed_input('Value: ') - postconf.validate_value(value) - new_config[key] = value - return new_config - - -def _timed_input(prompt): - # Set timeout - signal.alarm(3) - result = input(prompt) - # Disable timeout - signal.alarm(0) - return result - - -def _timeout_handler(signum, frame): - # https://stackoverflow.com/a/1336751 - logger.critical('[Time out]') - sys.exit(EXIT_TIMEOUT) +@reserved_for_root +def ipc_set_submission(): + audit.ldap.action_set_submission() if __name__ == '__main__': diff --git a/plinth/modules/email_server/__init__.py b/plinth/modules/email_server/__init__.py index e8e78d0a2..72bdfba3f 100644 --- a/plinth/modules/email_server/__init__.py +++ b/plinth/modules/email_server/__init__.py @@ -13,7 +13,7 @@ from plinth.modules.firewall.components import Firewall from . import audit from . import manifest -version = 1 +version = 31 managed_packages = ['postfix', 'dovecot-pop3d', 'dovecot-imapd', 'dovecot-lmtpd', 'dovecot-ldap', 'dovecot-managesieved'] managed_services = ['postfix', 'dovecot'] @@ -86,8 +86,8 @@ class EmailServerApp(plinth.app.App): def diagnose(self): """Run diagnostics and return the results""" results = super().diagnose() - results.append(audit.domain.get().summarize()) - results.append(audit.ldap.get().summarize()) + results.extend([r.summarize() for r in audit.domain.get()]) + results.extend([r.summarize() for r in audit.ldap.get()]) return results diff --git a/plinth/modules/email_server/audit/domain.py b/plinth/modules/email_server/audit/domain.py index 65815a132..194002637 100644 --- a/plinth/modules/email_server/audit/domain.py +++ b/plinth/modules/email_server/audit/domain.py @@ -6,7 +6,7 @@ from . import models def get(): # Stub - return models.Result('Email domains') + return [models.Result('Email domains')] def repair(): diff --git a/plinth/modules/email_server/audit/ldap.py b/plinth/modules/email_server/audit/ldap.py index 4da939640..10123eec7 100644 --- a/plinth/modules/email_server/audit/ldap.py +++ b/plinth/modules/email_server/audit/ldap.py @@ -1,4 +1,8 @@ +"""Audit of LDAP and mail submission settings""" # SPDX-License-Identifier: AGPL-3.0-or-later + +from plinth import actions + import plinth.modules.email_server.postconf as postconf from . import models @@ -8,18 +12,49 @@ default_config = { 'smtpd_sasl_path': 'private/auth' } +submission_flags = postconf.ServiceFlags( + service='submission', type='inet', private='n', unpriv='-', chroot='y', + wakeup='-', maxproc='-', command_args='smtpd' +) + +default_submission_options = { + 'syslog_name': 'postfix/submission', + 'smtpd_tls_security_level': 'encrypt', + 'smtpd_client_restrictions': 'permit_sasl_authenticated,reject', + 'smtpd_relay_restrictions': 'permit_sasl_authenticated,reject' +} + -# GET /audit/domain def get(): - """Compare current values with the default. Generate an audit report""" - results = models.Result('LDAP for user accounts') + """Compare current values with the default. Generate an audit report + + Recommended endpoint name: + GET /audit/ldap + """ + results = models.Result('Postfix uses Dovecot for SASL authentication') current_config = postconf.get_many(list(default_config.keys())) for key, value in default_config.items(): if current_config[key] != value: results.fails.append('{} should equal {}'.format(key, value)) - return results + return [results] -# POST /audit/domain/repair def repair(): + """Tries to repair LDAP and mail submission settings + + Recommended endpoint name: + POST /audit/ldap/repair + """ + actions.superuser_run('email_server', ['ipc', 'set_sasl']) + actions.superuser_run('email_server', ['ipc', 'set_submission']) + + +def action_set_sasl(): + """Called by email_server ipc set_sasl""" postconf.set_many(default_config) + + +def action_set_submission(): + """Called by email_server ipc set_submission""" + postconf.set_master_cf_options(service_flags=submission_flags, + options=default_submission_options) diff --git a/plinth/modules/email_server/postconf.py b/plinth/modules/email_server/postconf.py index b4dd8fee7..575668913 100644 --- a/plinth/modules/email_server/postconf.py +++ b/plinth/modules/email_server/postconf.py @@ -1,6 +1,7 @@ """Postconf wrapper providing thread-safe operations""" # SPDX-License-Identifier: AGPL-3.0-or-later +import dataclasses import re import subprocess import plinth.actions @@ -9,38 +10,67 @@ from .lock import Mutex postconf_mutex = Mutex('plinth-email-postconf.lock') +@dataclasses.dataclass +class ServiceFlags: + service: str + type: str + private: str + unpriv: str + chroot: str + wakeup: str + maxproc: str + command_args: str + + def serialize(self) -> str: + return ' '.join([self.service, self.type, self.private, self.unpriv, + self.chroot, self.wakeup, self.maxproc, + self.command_args]) + + def get_many(key_list): """Acquire resource lock. Get the list of postconf values as specified. Return a key-value map""" result = {} + for key in key_list: + validate_key(key) with postconf_mutex.lock_all(): for key in key_list: - result[key] = get_no_lock(key) + result[key] = get_unsafe(key) return result def set_many(kv_map): """Acquire resource lock. Set the list of postconf values as specified""" - # Encode email_server ipc input - lines = [] for key, value in kv_map.items(): validate_key(key) validate_value(value) - lines.append(key) - lines.append(value) - lines.append('\n') - stdin = '\n'.join(lines).encode('utf-8') - # Run action script as root - args = ['ipc', 'postconf_set_many_v1'] - with postconf_mutex.lock_threads_only(): - # The action script will take care of file locking - plinth.actions.superuser_run('email_server', args, input=stdin) + with postconf_mutex.lock_all(): + for key, value in kv_map.items(): + set_unsafe(key, value) -def get_no_lock(key): - """Get postconf value (no locking)""" - validate_key(key) +def set_master_cf_options(service_flags, options): + """Acquire resource lock. Set master.cf service options""" + if not isinstance(service_flags, ServiceFlags): + raise TypeError('service_flags') + for key, value in options.items(): + validate_key(key) + validate_value(value) + + service_slash_type = service_flags.service + '/' + service_flags.type + flag_string = service_flags.serialize() + + with postconf_mutex.lock_all(): + # /sbin/postconf -M "service/type=flag_string" + set_unsafe(service_slash_type, flag_string, '-M') + for short_key, value in options.items(): + # /sbin/postconf -P "service/type/short_key=value" + set_unsafe(service_slash_type + '/' + short_key, value, '-P') + + +def get_unsafe(key): + """Get postconf value (no locking, no sanitization)""" result = _run(['/sbin/postconf', key]) match = key + ' = ' if not result.startswith(match): @@ -48,11 +78,12 @@ def get_no_lock(key): return result[len(match):].strip() -def set_no_lock_assuming_root(key, value): - """Set postconf value (assuming root and no locking)""" - validate_key(key) - validate_value(value) - _run(['/sbin/postconf', '{}={}'.format(key, value)]) +def set_unsafe(key, value, flag=''): + """Set postconf value (assuming root, no locking, no sanitization)""" + if flag: + _run(['/sbin/postconf', flag, '{}={}'.format(key, value)]) + else: + _run(['/sbin/postconf', '{}={}'.format(key, value)]) def _run(args): From ba179a860e32721a17da9d0aece5f84a78dc7743 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Mon, 21 Jun 2021 01:02:29 +0000 Subject: [PATCH 08/56] email: Set up local delivery (no spam filtering) Hosts file: 192.168.56.101 host1.test.example virtualbox 10.42.0.101 host2.test.example freedombox Container setup (run commands in container) 1. Install the email server module 2. Create a FreedomBox account for `fred` 3. $ sudo postconf myhostname=host2.test.example $ echo "contactfred: fred" | sudo tee -a /etc/aliases $ sudo newaliases $ sudo systemctl reload postfix Test IMAP (run on host1.test.example) $ swaks --to=contactfred@host2.test.example --from=spam@host1.test.example $ mutt -f imaps://fred@host2.test.example Test mail submission (run in container): $ swaks --to=spam@host1.test.example --from=contactfred@host2.test.example \ --server host2.test.example:587 --tlso --auth-user=fred --- plinth/modules/email_server/__init__.py | 4 ++++ plinth/modules/email_server/audit/ldap.py | 4 +++- .../dovecot/conf.d/05-freedombox-auth.conf | 22 +++++++++++++++++++ .../dovecot/conf.d/90-freedombox-master.conf | 18 +++++++++++++++ .../dovecot/freedombox-ldap-passdb.conf.ext | 8 +++++++ .../dovecot/freedombox-ldap-userdb.conf.ext | 18 +++++++++++++++ 6 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 plinth/modules/email_server/data/etc/dovecot/conf.d/05-freedombox-auth.conf create mode 100644 plinth/modules/email_server/data/etc/dovecot/conf.d/90-freedombox-master.conf create mode 100644 plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-passdb.conf.ext create mode 100644 plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-userdb.conf.ext diff --git a/plinth/modules/email_server/__init__.py b/plinth/modules/email_server/__init__.py index 72bdfba3f..ea1e02aeb 100644 --- a/plinth/modules/email_server/__init__.py +++ b/plinth/modules/email_server/__init__.py @@ -8,6 +8,7 @@ import plinth.app import plinth.daemon import plinth.frontpage import plinth.menu +from plinth import actions from plinth.modules.firewall.components import Firewall from . import audit @@ -95,3 +96,6 @@ def setup(helper, old_version=None): """Installs and configures module""" helper.install(managed_packages) helper.call('post', audit.ldap.repair) + helper.call('post', app.enable) + for service_name in managed_services: + actions.superuser_run('service', ['reload', service_name]) diff --git a/plinth/modules/email_server/audit/ldap.py b/plinth/modules/email_server/audit/ldap.py index 10123eec7..3b2376789 100644 --- a/plinth/modules/email_server/audit/ldap.py +++ b/plinth/modules/email_server/audit/ldap.py @@ -9,7 +9,9 @@ from . import models default_config = { 'smtpd_sasl_auth_enable': 'yes', 'smtpd_sasl_type': 'dovecot', - 'smtpd_sasl_path': 'private/auth' + 'smtpd_sasl_path': 'private/auth', + 'mailbox_transport': 'lmtp:unix:private/dovecot-lmtp', + 'virtual_transport': 'lmtp:unix:private/dovecot-lmtp' } submission_flags = postconf.ServiceFlags( diff --git a/plinth/modules/email_server/data/etc/dovecot/conf.d/05-freedombox-auth.conf b/plinth/modules/email_server/data/etc/dovecot/conf.d/05-freedombox-auth.conf new file mode 100644 index 000000000..b8b027b42 --- /dev/null +++ b/plinth/modules/email_server/data/etc/dovecot/conf.d/05-freedombox-auth.conf @@ -0,0 +1,22 @@ +# Direct edits to this file will be lost! +# Manage your settings on Plinth + +auth_mechanisms = plain login + +passdb { + driver = ldap + args = /etc/dovecot/freedombox-ldap-passdb.conf.ext + # Block the default passdb lookup (pam, 10-auth.conf) + result_failure = return-fail + result_internalfail = return-fail + result_success = return-ok +} + +userdb { + driver = ldap + args = /etc/dovecot/freedombox-ldap-userdb.conf.ext + # Block the default userdb lookup (passwd, 10-auth.conf) + result_failure = return-fail + result_internalfail = return-fail + result_success = return-ok +} diff --git a/plinth/modules/email_server/data/etc/dovecot/conf.d/90-freedombox-master.conf b/plinth/modules/email_server/data/etc/dovecot/conf.d/90-freedombox-master.conf new file mode 100644 index 000000000..0b1137be7 --- /dev/null +++ b/plinth/modules/email_server/data/etc/dovecot/conf.d/90-freedombox-master.conf @@ -0,0 +1,18 @@ +# Direct edits to this file will be lost! +# Manage your settings on Plinth + +service auth { + unix_listener /var/spool/postfix/private/auth { + mode = 0600 + user = postfix + group = postfix + } +} + +service lmtp { + unix_listener /var/spool/postfix/private/dovecot-lmtp { + mode = 0600 + user = postfix + group = postfix + } +} diff --git a/plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-passdb.conf.ext b/plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-passdb.conf.ext new file mode 100644 index 000000000..0480c3354 --- /dev/null +++ b/plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-passdb.conf.ext @@ -0,0 +1,8 @@ +# Direct edits to this file will be lost! +# Manage your settings on Plinth + +uris = ldap://127.0.0.1 +base = dc=thisbox + +auth_bind = yes +auth_bind_userdn = uid=%u,ou=users,dc=thisbox diff --git a/plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-userdb.conf.ext b/plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-userdb.conf.ext new file mode 100644 index 000000000..c115853ed --- /dev/null +++ b/plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-userdb.conf.ext @@ -0,0 +1,18 @@ +# Direct edits to this file will be lost! +# Manage your settings on Plinth + +uris = ldap://127.0.0.1 +base = dc=thisbox + +user_attrs = \ + =home=%{ldap:homeDirectory}, \ + =uid=%{ldap:uidNumber}, \ + =gid=%{ldap:gidNumber}, \ + =mail=maildir:~/Maildir:LAYOUT=index + +user_filter = (&(objectClass=posixAccount)(uid=%Ln)) + +# For doveadm + +iterate_attrs = =user=%{ldap:uid} +iterate_filter = (objectClass=posixAccount) From 10c3a667b6d0408359c144921c01f0050d59a598 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Mon, 21 Jun 2021 23:50:23 +0000 Subject: [PATCH 09/56] email: Code quality fixes - flake8: Delete unused imports - lock.Mutex: Set file mode (rw-rw----) and ownership (plinth:plinth) --- plinth/modules/email_server/__init__.py | 2 +- plinth/modules/email_server/lock.py | 50 ++++++++++++++++++++++--- plinth/modules/email_server/postconf.py | 1 - 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/plinth/modules/email_server/__init__.py b/plinth/modules/email_server/__init__.py index ea1e02aeb..7e9fcbaca 100644 --- a/plinth/modules/email_server/__init__.py +++ b/plinth/modules/email_server/__init__.py @@ -14,7 +14,7 @@ from plinth.modules.firewall.components import Firewall from . import audit from . import manifest -version = 31 +version = 1 managed_packages = ['postfix', 'dovecot-pop3d', 'dovecot-imapd', 'dovecot-lmtpd', 'dovecot-ldap', 'dovecot-managesieved'] managed_services = ['postfix', 'dovecot'] diff --git a/plinth/modules/email_server/lock.py b/plinth/modules/email_server/lock.py index bdf138ba7..27f21c124 100644 --- a/plinth/modules/email_server/lock.py +++ b/plinth/modules/email_server/lock.py @@ -1,8 +1,11 @@ # SPDX-License-Identifier: AGPL-3.0-or-later import contextlib +import errno import fcntl import os +import pwd import threading +import time class Mutex: @@ -25,15 +28,50 @@ class Mutex: def lock_all(self): """Acquire both the thread lock and the file lock""" with self.lock_threads_only(): - fd = open(self.lock_path, 'wb') - # FIXME: Who can lock? - try: - os.fchmod(fd.fileno(), 0o666) # rw-rw-rw- - except OSError: - pass + # Set up + fd = self._open_lock_file() fcntl.lockf(fd, fcntl.LOCK_EX) + self._chmod_and_chown(fd) + # Enter context try: yield finally: + # Clean up fcntl.lockf(fd, fcntl.LOCK_UN) fd.close() + + def _open_lock_file(self): + """Attempt to open lock file for R&W. Raises OSError on failure""" + attempts = 10 + errno = -1 + fd = None + # Simulate a spin lock + while attempts > 0: + errno, fd = self._try(lambda: open(self.lock_path, 'wb')) + if errno == 0: + return fd + else: + attempts -= 1 + time.sleep(0.25) + raise OSError(errno, os.strerror(errno)) + + def _chmod_and_chown(self, fd): + """If the process UID is root, set fd's mode and ownership to + appropriate values. If we are not root, only set the mode""" + if os.getuid() == 0: + user_info = pwd.getpwnam('plinth') + os.fchown(fd.fileno(), 0, 0) + os.fchmod(fd.fileno(), 0o660) # rw-rw---- + fd.truncate(0) + os.fchown(fd.fileno(), user_info.pw_uid, user_info.pw_gid) + else: + self._try(lambda: os.fchmod(fd.fileno(), 0o660)) # rw-rw---- + + def _try(self, function): + try: + return 0, function() + except OSError as error: + if error.errno in (errno.EACCES, errno.EPERM): + return error.errno, None + else: + raise diff --git a/plinth/modules/email_server/postconf.py b/plinth/modules/email_server/postconf.py index 575668913..59b80d19a 100644 --- a/plinth/modules/email_server/postconf.py +++ b/plinth/modules/email_server/postconf.py @@ -4,7 +4,6 @@ import dataclasses import re import subprocess -import plinth.actions from .lock import Mutex postconf_mutex = Mutex('plinth-email-postconf.lock') From f20929c23f7e1ecce159529dc28c58d2a65564e7 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Sun, 27 Jun 2021 02:15:32 +0000 Subject: [PATCH 10/56] email: Fix enabling SMTPS; check return value - master.cf: Enable SMTPS - lock.Mutex: check the return value of lock.acquire - Write debug logs --- plinth/modules/email_server/audit/ldap.py | 24 +++++++++++++++++++++ plinth/modules/email_server/audit/models.py | 2 +- plinth/modules/email_server/lock.py | 11 ++++++++-- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/plinth/modules/email_server/audit/ldap.py b/plinth/modules/email_server/audit/ldap.py index 3b2376789..f5cf9bf3a 100644 --- a/plinth/modules/email_server/audit/ldap.py +++ b/plinth/modules/email_server/audit/ldap.py @@ -1,6 +1,8 @@ """Audit of LDAP and mail submission settings""" # SPDX-License-Identifier: AGPL-3.0-or-later +import logging + from plinth import actions import plinth.modules.email_server.postconf as postconf @@ -26,6 +28,20 @@ default_submission_options = { 'smtpd_relay_restrictions': 'permit_sasl_authenticated,reject' } +smtps_flags = postconf.ServiceFlags( + service='smtps', type='inet', private='n', unpriv='-', chroot='y', + wakeup='-', maxproc='-', command_args='smtpd' +) + +default_smtps_options = { + 'syslog_name': 'postfix/smtps', + 'smtpd_tls_wrappermode': 'yes', + 'smtpd_sasl_auth_enable': 'yes', + 'smtpd_relay_restrictions': 'permit_sasl_authenticated,reject' +} + +logger = logging.getLogger(__name__) + def get(): """Compare current values with the default. Generate an audit report @@ -47,7 +63,13 @@ def repair(): Recommended endpoint name: POST /audit/ldap/repair """ + logger.debug('Updating postconf: %r', default_config) actions.superuser_run('email_server', ['ipc', 'set_sasl']) + + logger.debug('Setting up postfix %s service in master.cf: %r', + submission_flags.service, default_submission_options) + logger.debug('And postfix %s service: %r', smtps_flags.service, + default_smtps_options) actions.superuser_run('email_server', ['ipc', 'set_submission']) @@ -60,3 +82,5 @@ def action_set_submission(): """Called by email_server ipc set_submission""" postconf.set_master_cf_options(service_flags=submission_flags, options=default_submission_options) + postconf.set_master_cf_options(service_flags=smtps_flags, + options=default_smtps_options) diff --git a/plinth/modules/email_server/audit/models.py b/plinth/modules/email_server/audit/models.py index ec91a9280..de64a36c5 100644 --- a/plinth/modules/email_server/audit/models.py +++ b/plinth/modules/email_server/audit/models.py @@ -25,7 +25,7 @@ class Result: def write_logs(self): """Log errors and failures""" - logger.debug('Ran audit: ' + self.title) + logger.debug('Ran audit: %s', self.title) for message in self.errors: logger.critical(message) for message in self.fails: diff --git a/plinth/modules/email_server/lock.py b/plinth/modules/email_server/lock.py index 27f21c124..0d812df9c 100644 --- a/plinth/modules/email_server/lock.py +++ b/plinth/modules/email_server/lock.py @@ -2,11 +2,14 @@ import contextlib import errno import fcntl +import logging import os import pwd import threading import time +logger = logging.getLogger(__name__) + class Mutex: """File and pthread lock based resource mutex""" @@ -18,7 +21,8 @@ class Mutex: @contextlib.contextmanager def lock_threads_only(self): """Acquire the thread lock but not the file lock""" - self.thread_mutex.acquire(timeout=5) + if not self.thread_mutex.acquire(timeout=5): + raise RuntimeError('Could not acquire thread lock') try: yield finally: @@ -65,7 +69,10 @@ class Mutex: fd.truncate(0) os.fchown(fd.fileno(), user_info.pw_uid, user_info.pw_gid) else: - self._try(lambda: os.fchmod(fd.fileno(), 0o660)) # rw-rw---- + errno, _ = self._try(lambda: os.fchmod(fd.fileno(), 0o660)) + if errno != 0: + logger.warning('chmod failed, lock path %s, errno %d', + self.lock_path, errno) def _try(self, function): try: From 81c9632f5ab6a370e0d581c5edcddd2109779c72 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Wed, 30 Jun 2021 21:19:55 +0000 Subject: [PATCH 11/56] email: dovecot: Support user lookup by UID number Dovecot refuses to open the root user's mail box Need (!(uidNumber=0)) to correctly return status code --- .../freedombox-ldap-userdb-aliases.conf.ext | 20 +++++++++++++++++++ .../dovecot/freedombox-ldap-userdb.conf.ext | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-userdb-aliases.conf.ext diff --git a/plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-userdb-aliases.conf.ext b/plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-userdb-aliases.conf.ext new file mode 100644 index 000000000..c7eb3c369 --- /dev/null +++ b/plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-userdb-aliases.conf.ext @@ -0,0 +1,20 @@ +# Direct edits to this file will be lost! +# Manage your settings on Plinth https://localhost/plinth/apps/email_server + +uris = ldap://127.0.0.1 +base = dc=thisbox + +user_attrs = \ + =home=%{ldap:homeDirectory}, \ + =uid=%{ldap:uidNumber}, \ + =gid=%{ldap:gidNumber}, \ + =user=%{ldap:uid}, \ + =mail=maildir:~/Maildir:LAYOUT=index + +user_filter = \ + (&(objectClass=posixAccount)(!(uidNumber=0))(uidNumber=%n)(!(uid=%n))) + +# doveadm -A + +iterate_attrs = =user=%{ldap:uid} +iterate_filter = (objectClass=posixAccount) diff --git a/plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-userdb.conf.ext b/plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-userdb.conf.ext index c115853ed..d6e464def 100644 --- a/plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-userdb.conf.ext +++ b/plinth/modules/email_server/data/etc/dovecot/freedombox-ldap-userdb.conf.ext @@ -10,7 +10,7 @@ user_attrs = \ =gid=%{ldap:gidNumber}, \ =mail=maildir:~/Maildir:LAYOUT=index -user_filter = (&(objectClass=posixAccount)(uid=%Ln)) +user_filter = (&(objectClass=posixAccount)(uid=%Ln)(!(uidNumber=0))) # For doveadm From 758c8791f1ce37b41e9e574f19cd9aaa2335c5fe Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Thu, 1 Jul 2021 07:06:54 +0000 Subject: [PATCH 12/56] email: Address some code review comments - __init__.py: Changed email server description - audit module: Added module docstring - email_server action: - Used argparse - Replaced "wrapper functions" with a getattr based lookup method --- actions/email_server | 33 +++++++++++-------- plinth/modules/email_server/__init__.py | 2 +- plinth/modules/email_server/audit/__init__.py | 4 +++ plinth/modules/email_server/audit/ldap.py | 10 +++--- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/actions/email_server b/actions/email_server index 4512367f5..a7c4d00ce 100755 --- a/actions/email_server +++ b/actions/email_server @@ -1,12 +1,11 @@ #!/usr/bin/python3 # SPDX-License-Identifier: AGPL-3.0-or-later +import argparse import logging import os import sys -import plinth.modules.email_server.audit as audit - logger = logging.getLogger(__name__) EXIT_SYNTAX = 10 @@ -23,23 +22,29 @@ def reserved_for_root(fun): def main(): - if len(sys.argv) != 3: - sys.exit(EXIT_SYNTAX) - if sys.argv[1] != 'ipc': - sys.exit(EXIT_SYNTAX) - - function_name = 'ipc_' + sys.argv[2] - globals()[function_name]() + parser = argparse.ArgumentParser() + parser.add_argument('ipc', nargs=3) + arguments = parser.parse_args() + subcommand_ipc(arguments) @reserved_for_root -def ipc_set_sasl(): - audit.ldap.action_set_sasl() +def subcommand_ipc(arguments): + import plinth.modules.email_server.audit as audit + _, module_name, action_name = arguments.ipc + # We only run actions defined in the audit module + if module_name not in audit.__all__: + logger.critical('Bad module name: %r', module_name) + sys.exit(EXIT_SYNTAX) -@reserved_for_root -def ipc_set_submission(): - audit.ldap.action_set_submission() + module = getattr(audit, module_name) + function = getattr(module, 'action_' + action_name, None) + if function is None: + logger.critical('Bad action: %s/%r', module_name, action_name) + sys.exit(EXIT_SYNTAX) + + function() if __name__ == '__main__': diff --git a/plinth/modules/email_server/__init__.py b/plinth/modules/email_server/__init__.py index 7e9fcbaca..7785bfc7e 100644 --- a/plinth/modules/email_server/__init__.py +++ b/plinth/modules/email_server/__init__.py @@ -33,7 +33,7 @@ class EmailServerApp(plinth.app.App): app_id=self.app_id, version=version, name=_('Email Server'), - short_description=_('An email server for FreedomBox'), + short_description=_('Powered by Postfix, Dovecot & Rspamd'), manual_page='EmailServer', clients=manifest.clients, donation_url='https://freedomboxfoundation.org/donate/' diff --git a/plinth/modules/email_server/audit/__init__.py b/plinth/modules/email_server/audit/__init__.py index d82a9cd89..80c90ceb5 100644 --- a/plinth/modules/email_server/audit/__init__.py +++ b/plinth/modules/email_server/audit/__init__.py @@ -1,4 +1,8 @@ # SPDX-License-Identifier: AGPL-3.0-or-later +""" +Provides diagnosis and repair of email server configuration issues +""" + from . import ldap from . import domain diff --git a/plinth/modules/email_server/audit/ldap.py b/plinth/modules/email_server/audit/ldap.py index f5cf9bf3a..c3f11df85 100644 --- a/plinth/modules/email_server/audit/ldap.py +++ b/plinth/modules/email_server/audit/ldap.py @@ -64,13 +64,11 @@ def repair(): POST /audit/ldap/repair """ logger.debug('Updating postconf: %r', default_config) - actions.superuser_run('email_server', ['ipc', 'set_sasl']) + actions.superuser_run('email_server', ['ipc', 'ldap', 'set_sasl']) - logger.debug('Setting up postfix %s service in master.cf: %r', - submission_flags.service, default_submission_options) - logger.debug('And postfix %s service: %r', smtps_flags.service, - default_smtps_options) - actions.superuser_run('email_server', ['ipc', 'set_submission']) + logger.debug('Setting up postfix services:\n %r\n %r', + default_submission_options, default_smtps_options) + actions.superuser_run('email_server', ['ipc', 'ldap', 'set_submission']) def action_set_sasl(): From 7166e63b02723dd6b6d73b8fcd52b24f448692b4 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Thu, 1 Jul 2021 09:34:15 +0000 Subject: [PATCH 13/56] email: Install rspamd; proxy its web interface --- plinth/modules/email_server/__init__.py | 10 ++++++- plinth/modules/email_server/audit/__init__.py | 3 ++- plinth/modules/email_server/audit/spam.py | 26 +++++++++++++++++++ .../email-server-freedombox.conf | 23 ++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 plinth/modules/email_server/audit/spam.py create mode 100644 plinth/modules/email_server/data/etc/apache2/conf-available/email-server-freedombox.conf diff --git a/plinth/modules/email_server/__init__.py b/plinth/modules/email_server/__init__.py index 7785bfc7e..450b034ca 100644 --- a/plinth/modules/email_server/__init__.py +++ b/plinth/modules/email_server/__init__.py @@ -9,6 +9,7 @@ import plinth.daemon import plinth.frontpage import plinth.menu from plinth import actions +from plinth.modules.apache.components import Webserver from plinth.modules.firewall.components import Firewall from . import audit @@ -16,7 +17,8 @@ from . import manifest version = 1 managed_packages = ['postfix', 'dovecot-pop3d', 'dovecot-imapd', - 'dovecot-lmtpd', 'dovecot-ldap', 'dovecot-managesieved'] + 'dovecot-lmtpd', 'dovecot-ldap', 'dovecot-managesieved', + 'rspamd'] managed_services = ['postfix', 'dovecot'] app = None @@ -50,6 +52,11 @@ class EmailServerApp(plinth.app.App): ) self.add(menu_item) + # /rspamd location + webserver = Webserver('webserver-email', 'email-server-freedombox', + urls=['https://{host}/rspamd']) + self.add(webserver) + shortcut = plinth.frontpage.Shortcut( 'shortcut_' + self.app_id, name=info.name, @@ -96,6 +103,7 @@ def setup(helper, old_version=None): """Installs and configures module""" helper.install(managed_packages) helper.call('post', audit.ldap.repair) + helper.call('post', audit.spam.repair) helper.call('post', app.enable) for service_name in managed_services: actions.superuser_run('service', ['reload', service_name]) diff --git a/plinth/modules/email_server/audit/__init__.py b/plinth/modules/email_server/audit/__init__.py index 80c90ceb5..41f1b99c6 100644 --- a/plinth/modules/email_server/audit/__init__.py +++ b/plinth/modules/email_server/audit/__init__.py @@ -5,5 +5,6 @@ Provides diagnosis and repair of email server configuration issues from . import ldap from . import domain +from . import spam -__all__ = ['ldap', 'domain'] +__all__ = ['ldap', 'domain', 'spam'] diff --git a/plinth/modules/email_server/audit/spam.py b/plinth/modules/email_server/audit/spam.py new file mode 100644 index 000000000..a7ec22318 --- /dev/null +++ b/plinth/modules/email_server/audit/spam.py @@ -0,0 +1,26 @@ +"""Configures spam filters and the virus scanner""" +# SPDX-License-Identifier: AGPL-3.0-or-later + +import logging + +from plinth import actions + +import plinth.modules.email_server.postconf as postconf + +milter_config = { + 'milter_mail_macros': 'i {auth_type} {auth_authen} {auth_author} '\ + '{client_addr} {client_name} {mail_addr} {mail_host} {mail_mailer}', + 'smtpd_milters': 'inet:127.0.0.1:11332', + 'non_smtpd_milters': 'inet:127.0.0.1:11332' +} + +logger = logging.getLogger(__name__) + + +def repair(): + logger.debug('Updating postconf: %r', milter_config) + actions.superuser_run('email_server', ['ipc', 'spam', 'set_filter']) + + +def action_set_filter(): + postconf.set_many(milter_config) diff --git a/plinth/modules/email_server/data/etc/apache2/conf-available/email-server-freedombox.conf b/plinth/modules/email_server/data/etc/apache2/conf-available/email-server-freedombox.conf new file mode 100644 index 000000000..f674fc07c --- /dev/null +++ b/plinth/modules/email_server/data/etc/apache2/conf-available/email-server-freedombox.conf @@ -0,0 +1,23 @@ + + Redirect "/rspamd/" + + + + ProxyPass http://127.0.0.1:11334/ + ProxyPassReverse http://127.0.0.1:11334/ + # Modify proxy headers + ProxyAddHeaders off + RequestHeader unset Forwarded + RequestHeader unset Via + RequestHeader unset X-Forwarded-For + RequestHeader unset X-Forwarded-Host + RequestHeader unset X-Forwarded-Proto + RequestHeader unset X-Forwarded-Server + RequestHeader unset X-Real-IP + RequestHeader set X-Forwarded-For "127.0.0.1" + # Require SSO + Include includes/freedombox-single-sign-on.conf + + TKTAuthToken "admin" + + From df14e7497266cdbe53ad586cb3096c9a3de4320c Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Fri, 2 Jul 2021 04:07:02 +0000 Subject: [PATCH 14/56] email: Parse command arguments with a mutually exclusive group --- actions/email_server | 27 ++++++++++++++++++----- plinth/modules/email_server/audit/ldap.py | 4 ++-- plinth/modules/email_server/audit/spam.py | 2 +- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/actions/email_server b/actions/email_server index a7c4d00ce..dad26f5d0 100755 --- a/actions/email_server +++ b/actions/email_server @@ -23,16 +23,22 @@ def reserved_for_root(fun): def main(): parser = argparse.ArgumentParser() - parser.add_argument('ipc', nargs=3) - arguments = parser.parse_args() - subcommand_ipc(arguments) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-i', nargs=2, dest='ipc') + group.add_argument('-t', nargs=1, dest='touch_file') + + adict = vars(parser.parse_args()) + generator = (kv for kv in adict.items() if kv[1] is not None) + subcommand, arguments = next(generator) + + function = globals()['subcommand_' + subcommand] + function(*arguments) @reserved_for_root -def subcommand_ipc(arguments): +def subcommand_ipc(module_name, action_name): import plinth.modules.email_server.audit as audit - _, module_name, action_name = arguments.ipc # We only run actions defined in the audit module if module_name not in audit.__all__: logger.critical('Bad module name: %r', module_name) @@ -47,5 +53,16 @@ def subcommand_ipc(arguments): function() +def subcommand_touch_file(path): + import pathlib + + if os.getuid == 0: + logger.critical('Do not run as root') + sys.exit(EXIT_PERM) + + # mode is influenced by umask + pathlib.Path(path).touch(mode=0o660, exist_ok=True) + + if __name__ == '__main__': main() diff --git a/plinth/modules/email_server/audit/ldap.py b/plinth/modules/email_server/audit/ldap.py index c3f11df85..7217b78eb 100644 --- a/plinth/modules/email_server/audit/ldap.py +++ b/plinth/modules/email_server/audit/ldap.py @@ -64,11 +64,11 @@ def repair(): POST /audit/ldap/repair """ logger.debug('Updating postconf: %r', default_config) - actions.superuser_run('email_server', ['ipc', 'ldap', 'set_sasl']) + actions.superuser_run('email_server', ['-i', 'ldap', 'set_sasl']) logger.debug('Setting up postfix services:\n %r\n %r', default_submission_options, default_smtps_options) - actions.superuser_run('email_server', ['ipc', 'ldap', 'set_submission']) + actions.superuser_run('email_server', ['-i', 'ldap', 'set_submission']) def action_set_sasl(): diff --git a/plinth/modules/email_server/audit/spam.py b/plinth/modules/email_server/audit/spam.py index a7ec22318..1b7b0ee34 100644 --- a/plinth/modules/email_server/audit/spam.py +++ b/plinth/modules/email_server/audit/spam.py @@ -19,7 +19,7 @@ logger = logging.getLogger(__name__) def repair(): logger.debug('Updating postconf: %r', milter_config) - actions.superuser_run('email_server', ['ipc', 'spam', 'set_filter']) + actions.superuser_run('email_server', ['-i', 'spam', 'set_filter']) def action_set_filter(): From 7397326d570abb57280753c4e597124147166f1a Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Fri, 2 Jul 2021 04:59:21 +0000 Subject: [PATCH 15/56] email: mutex: create lock file as plinth user --- plinth/modules/email_server/lock.py | 52 +++++++++++++---------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/plinth/modules/email_server/lock.py b/plinth/modules/email_server/lock.py index 0d812df9c..c3ff204a5 100644 --- a/plinth/modules/email_server/lock.py +++ b/plinth/modules/email_server/lock.py @@ -5,8 +5,8 @@ import fcntl import logging import os import pwd +import subprocess import threading -import time logger = logging.getLogger(__name__) @@ -35,7 +35,6 @@ class Mutex: # Set up fd = self._open_lock_file() fcntl.lockf(fd, fcntl.LOCK_EX) - self._chmod_and_chown(fd) # Enter context try: yield @@ -46,33 +45,30 @@ class Mutex: def _open_lock_file(self): """Attempt to open lock file for R&W. Raises OSError on failure""" - attempts = 10 - errno = -1 - fd = None - # Simulate a spin lock - while attempts > 0: - errno, fd = self._try(lambda: open(self.lock_path, 'wb')) - if errno == 0: - return fd - else: - attempts -= 1 - time.sleep(0.25) - raise OSError(errno, os.strerror(errno)) - - def _chmod_and_chown(self, fd): - """If the process UID is root, set fd's mode and ownership to - appropriate values. If we are not root, only set the mode""" if os.getuid() == 0: - user_info = pwd.getpwnam('plinth') - os.fchown(fd.fileno(), 0, 0) - os.fchmod(fd.fileno(), 0o660) # rw-rw---- - fd.truncate(0) - os.fchown(fd.fileno(), user_info.pw_uid, user_info.pw_gid) - else: - errno, _ = self._try(lambda: os.fchmod(fd.fileno(), 0o660)) - if errno != 0: - logger.warning('chmod failed, lock path %s, errno %d', - self.lock_path, errno) + if os.path.exists(self.lock_path): + # Check its owner and mode + stat = os.stat(self.lock_path) + owner = pwd.getpwuid(stat.st_uid).pw_name + mode = stat.st_mode & 0o777 + if owner != 'plinth' or mode != 0o600: + logger.warning('Clean up bad file %s', self.lock_path) + os.unlink(self.lock_path) + self._create_lock_file_as_plinth() + else: + self._create_lock_file_as_plinth() + + return open(self.lock_path, 'wb+') + + def _create_lock_file_as_plinth(self): + args = ['sudo', '-n', '-u', 'plinth', 'touch', self.lock_path] + completed = subprocess.run(args, capture_output=True) + if completed.returncode != 0: + logger.critical('Process returned %d', completed.returncode) + logger.critical('Stdout: %r', completed.stdout) + logger.critical('Stderr: %r', completed.stderr) + raise OSError('Could not create ' + self.lock_path) + os.chmod(self.lock_path, 0o600) def _try(self, function): try: From aab3fe9c0220e8875ac2dbdc3884eb3a3241d5b3 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Mon, 5 Jul 2021 21:27:18 +0000 Subject: [PATCH 16/56] email, plinth.log: Write more information to syslog sudo journalctl -b -o short-monotonic --no-pager -f --- actions/email_server | 33 +++++++++++++++++++++++++++++---- plinth/log.py | 23 +++++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/actions/email_server b/actions/email_server index dad26f5d0..39b39610c 100755 --- a/actions/email_server +++ b/actions/email_server @@ -6,11 +6,15 @@ import logging import os import sys -logger = logging.getLogger(__name__) +import plinth.log EXIT_SYNTAX = 10 EXIT_PERM = 20 +# Set up logging +plinth.log.pipe_to_syslog(to_stderr='tty') +logger = logging.getLogger(os.path.basename(__file__)) + def reserved_for_root(fun): def wrapped(*args, **kwargs): @@ -22,6 +26,10 @@ def reserved_for_root(fun): def main(): + if not sys.stdin.isatty(): + print('WARNING: Output will not be shown. Check syslog for logs', + file=sys.stderr) + parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-i', nargs=2, dest='ipc') @@ -32,7 +40,12 @@ def main(): subcommand, arguments = next(generator) function = globals()['subcommand_' + subcommand] - function(*arguments) + try: + function(*arguments) + except Exception as e: + logger.exception(e) + _log_additional_info() + sys.exit(1) @reserved_for_root @@ -56,13 +69,25 @@ def subcommand_ipc(module_name, action_name): def subcommand_touch_file(path): import pathlib - if os.getuid == 0: - logger.critical('Do not run as root') + if os.getuid() == 0: + logger.critical('Do not run the `-t` option as root') sys.exit(EXIT_PERM) # mode is influenced by umask pathlib.Path(path).touch(mode=0o660, exist_ok=True) +def _log_additional_info(): + import grp + import pwd + resu = ','.join(pwd.getpwuid(uid).pw_name for uid in os.getresuid()) + resg = ','.join(grp.getgrgid(gid).gr_name for gid in os.getresgid()) + pyver = sys.version.replace('\n', ' ') + logger.error('--- Additional Information ---') + logger.error('resuid=%s, resgid=%s', resu, resg) + logger.error('argv=%r, cwd=%r', sys.argv, os.getcwd()) + logger.error('pyver=%s (%s)', pyver, os.uname().machine) + + if __name__ == '__main__': main() diff --git a/plinth/log.py b/plinth/log.py index 0d0a316c4..ddf984321 100644 --- a/plinth/log.py +++ b/plinth/log.py @@ -5,6 +5,8 @@ Setup logging for the application. import importlib import logging +import logging.handlers +import sys import warnings import cherrypy @@ -127,3 +129,24 @@ def get_configuration(): configuration['root']['handlers'].append('journal') return configuration + + +def pipe_to_syslog(level=logging.INFO, to_stderr=True): + """Make the root logger write to syslog and stderr. Useful in actions""" + logger = logging.getLogger() + logger.setLevel(level) + + fmt = '/freedombox/%(name)s[%(process)d]: %(levelname)s: %(message)s' + formatter = logging.Formatter(fmt=fmt) + + # Using syslog in Python: https://stackoverflow.com/q/3968669 + syslog_handler = logging.handlers.SysLogHandler(address='/dev/log') + syslog_handler.setFormatter(formatter) + logger.addHandler(syslog_handler) + + if to_stderr == 'tty' and sys.stdin.isatty(): + to_stderr = True + if to_stderr is True: + stderr_handler = logging.StreamHandler() + stderr_handler.setFormatter(formatter) + logger.addHandler(stderr_handler) From 573287cf287907610f5c8514600a5bc70f7c6457 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Mon, 5 Jul 2021 23:30:09 +0000 Subject: [PATCH 17/56] email: postconf: Handle postconf returning an empty key However, if /sbin/postconf complained, a KeyError will be raised --- plinth/modules/email_server/postconf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plinth/modules/email_server/postconf.py b/plinth/modules/email_server/postconf.py index 59b80d19a..4762bd522 100644 --- a/plinth/modules/email_server/postconf.py +++ b/plinth/modules/email_server/postconf.py @@ -71,7 +71,7 @@ def set_master_cf_options(service_flags, options): def get_unsafe(key): """Get postconf value (no locking, no sanitization)""" result = _run(['/sbin/postconf', key]) - match = key + ' = ' + match = key + ' =' if not result.startswith(match): raise KeyError(key) return result[len(match):].strip() From e2535bad496f759f97a63876e12c0c83ea33b1b7 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Tue, 6 Jul 2021 00:01:06 +0000 Subject: [PATCH 18/56] email: audit: improve the speed of post-installation setup - New class: `MainCfDiagnosis` - "Advise and repair" mechanism reduces the number of postconf calls - File locking: lock acquisition moved into audit module - Enables finer-grained control --- plinth/modules/email_server/__init__.py | 1 + plinth/modules/email_server/audit/domain.py | 2 +- plinth/modules/email_server/audit/ldap.py | 35 +++++++++----- plinth/modules/email_server/audit/models.py | 51 ++++++++++++++++++++- plinth/modules/email_server/audit/spam.py | 34 ++++++++++++-- plinth/modules/email_server/postconf.py | 20 +++++--- 6 files changed, 119 insertions(+), 24 deletions(-) diff --git a/plinth/modules/email_server/__init__.py b/plinth/modules/email_server/__init__.py index 450b034ca..c8d6faff1 100644 --- a/plinth/modules/email_server/__init__.py +++ b/plinth/modules/email_server/__init__.py @@ -96,6 +96,7 @@ class EmailServerApp(plinth.app.App): results = super().diagnose() results.extend([r.summarize() for r in audit.domain.get()]) results.extend([r.summarize() for r in audit.ldap.get()]) + results.extend([r.summarize() for r in audit.spam.get()]) return results diff --git a/plinth/modules/email_server/audit/domain.py b/plinth/modules/email_server/audit/domain.py index 194002637..1f831b433 100644 --- a/plinth/modules/email_server/audit/domain.py +++ b/plinth/modules/email_server/audit/domain.py @@ -6,7 +6,7 @@ from . import models def get(): # Stub - return [models.Result('Email domains')] + return [models.Diagnosis('Email domains')] def repair(): diff --git a/plinth/modules/email_server/audit/ldap.py b/plinth/modules/email_server/audit/ldap.py index 7217b78eb..790b8a953 100644 --- a/plinth/modules/email_server/audit/ldap.py +++ b/plinth/modules/email_server/audit/ldap.py @@ -49,12 +49,10 @@ def get(): Recommended endpoint name: GET /audit/ldap """ - results = models.Result('Postfix uses Dovecot for SASL authentication') - current_config = postconf.get_many(list(default_config.keys())) - for key, value in default_config.items(): - if current_config[key] != value: - results.fails.append('{} should equal {}'.format(key, value)) - return [results] + results = [] + with postconf.postconf_mutex.lock_all(): + results.append(check_sasl()) + return results def repair(): @@ -63,22 +61,35 @@ def repair(): Recommended endpoint name: POST /audit/ldap/repair """ - logger.debug('Updating postconf: %r', default_config) actions.superuser_run('email_server', ['-i', 'ldap', 'set_sasl']) - - logger.debug('Setting up postfix services:\n %r\n %r', - default_submission_options, default_smtps_options) actions.superuser_run('email_server', ['-i', 'ldap', 'set_submission']) +def check_sasl(): + diagnosis = models.MainCfDiagnosis('Postfix-Dovecot SASL integration') + current = postconf.get_many_unsafe(default_config.keys()) + diagnosis.compare_and_advise(current=current, default=default_config) + return diagnosis + + +def fix_sasl(diagnosis): + diagnosis.assert_resolved() + logger.info('Setting postconf: %r', diagnosis.advice) + postconf.set_many_unsafe(diagnosis.advice) + + def action_set_sasl(): - """Called by email_server ipc set_sasl""" - postconf.set_many(default_config) + """Called by email_server ipc ldap set_sasl""" + with postconf.postconf_mutex.lock_all(): + fix_sasl(check_sasl()) def action_set_submission(): """Called by email_server ipc set_submission""" + logger.info('Set postfix service: %r', default_submission_options) postconf.set_master_cf_options(service_flags=submission_flags, options=default_submission_options) + + logger.info('Set postfix service: %r', default_smtps_options) postconf.set_master_cf_options(service_flags=smtps_flags, options=default_smtps_options) diff --git a/plinth/modules/email_server/audit/models.py b/plinth/modules/email_server/audit/models.py index de64a36c5..a08efb0a2 100644 --- a/plinth/modules/email_server/audit/models.py +++ b/plinth/modules/email_server/audit/models.py @@ -5,12 +5,30 @@ import logging logger = logging.getLogger(__name__) -class Result: +class UnresolvedIssueError(AssertionError): + pass + + +class Diagnosis: def __init__(self, title): self.title = title self.fails = [] self.errors = [] + def critical(self, message_fmt, *args): + """Append a message to the fails list""" + if args: + self.fails.append(message_fmt % args) + else: + self.fails.append(message_fmt) + + def error(self, message_fmt, *args): + """Append a message to the errors list""" + if args: + self.errors.append(message_fmt % args) + else: + self.errors.append(message_fmt) + def summarize(self, log=True): """Return a 2-element list for the diagnose function in AppView""" if log: @@ -30,3 +48,34 @@ class Result: logger.critical(message) for message in self.fails: logger.error(message) + + +class MainCfDiagnosis(Diagnosis): + def __init__(self, title): + super().__init__(title) + self.advice = {} + self.user = {} + + def flag(self, key, corrected_value=None, user=None): + self.advice[key] = corrected_value + self.user[key] = user + + def unresolved_issues(self): + """Returns an interator of dictionary keys""" + for key, value in self.advice.items(): + if value is None: + yield key + + def compare_and_advise(self, current, default): + if len(current) > len(default): + raise ValueError('Sanity check failed: dictionary sizes') + for key, value in default.items(): + if current.get(key, None) != value: + self.flag(key, corrected_value=value) + self.critical('%s must equal %s', key, value) + + def assert_resolved(self): + """Raises an UnresolvedIssueError if the diagnosis report contains an + unresolved issue""" + if None in self.advice.values(): + raise UnresolvedIssueError('Assertion failed') diff --git a/plinth/modules/email_server/audit/spam.py b/plinth/modules/email_server/audit/spam.py index 1b7b0ee34..f0ad26fe3 100644 --- a/plinth/modules/email_server/audit/spam.py +++ b/plinth/modules/email_server/audit/spam.py @@ -6,21 +6,47 @@ import logging from plinth import actions import plinth.modules.email_server.postconf as postconf +from . import models milter_config = { - 'milter_mail_macros': 'i {auth_type} {auth_authen} {auth_author} '\ - '{client_addr} {client_name} {mail_addr} {mail_host} {mail_mailer}', + 'milter_mail_macros': 'i ' + ' '.join([ + '{auth_type}', '{auth_authen}', '{auth_author}', + '{client_addr}', '{client_name}', + '{mail_addr}', '{mail_host}', '{mail_mailer}' + ]), + # XXX In postconf this field is a list 'smtpd_milters': 'inet:127.0.0.1:11332', + # XXX In postconf this field is a list 'non_smtpd_milters': 'inet:127.0.0.1:11332' } logger = logging.getLogger(__name__) +def get(): + results = [] + with postconf.postconf_mutex.lock_all(): + results.append(check_filter()) + return results + + def repair(): - logger.debug('Updating postconf: %r', milter_config) actions.superuser_run('email_server', ['-i', 'spam', 'set_filter']) +def check_filter(): + diagnosis = models.MainCfDiagnosis('Postfix milter') + current = postconf.get_many_unsafe(milter_config.keys()) + diagnosis.compare_and_advise(current=current, default=milter_config) + return diagnosis + + +def fix_filter(diagnosis): + diagnosis.assert_resolved() + logger.info('Setting postconf: %r', diagnosis.advice) + postconf.set_many_unsafe(diagnosis.advice) + + def action_set_filter(): - postconf.set_many(milter_config) + with postconf.postconf_mutex.lock_all(): + fix_filter(check_filter()) diff --git a/plinth/modules/email_server/postconf.py b/plinth/modules/email_server/postconf.py index 4762bd522..2553f24d3 100644 --- a/plinth/modules/email_server/postconf.py +++ b/plinth/modules/email_server/postconf.py @@ -29,13 +29,17 @@ class ServiceFlags: def get_many(key_list): """Acquire resource lock. Get the list of postconf values as specified. Return a key-value map""" - result = {} for key in key_list: validate_key(key) with postconf_mutex.lock_all(): - for key in key_list: - result[key] = get_unsafe(key) - return result + return get_many_unsafe(key_list) + + +def get_many_unsafe(key_iterator): + result = {} + for key in key_iterator: + result[key] = get_unsafe(key) + return result def set_many(kv_map): @@ -45,8 +49,12 @@ def set_many(kv_map): validate_value(value) with postconf_mutex.lock_all(): - for key, value in kv_map.items(): - set_unsafe(key, value) + set_many_unsafe(kv_map) + + +def set_many_unsafe(kv_map): + for key, value in kv_map.items(): + set_unsafe(key, value) def set_master_cf_options(service_flags, options): From 91c907f65700b2829fc7c4a192c81b8c4e24df47 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Wed, 7 Jul 2021 21:51:23 +0000 Subject: [PATCH 19/56] email: Open lock file as plinth user --- plinth/modules/email_server/lock.py | 83 ++++++++++++++++--------- plinth/modules/email_server/postconf.py | 2 +- 2 files changed, 54 insertions(+), 31 deletions(-) diff --git a/plinth/modules/email_server/lock.py b/plinth/modules/email_server/lock.py index c3ff204a5..f613db75e 100644 --- a/plinth/modules/email_server/lock.py +++ b/plinth/modules/email_server/lock.py @@ -1,32 +1,43 @@ # SPDX-License-Identifier: AGPL-3.0-or-later import contextlib -import errno import fcntl import logging import os import pwd +import re import subprocess import threading +lock_name_pattern = re.compile('^[0-9a-zA-Z_-]+$') logger = logging.getLogger(__name__) +class RaceCondition(AssertionError): + pass + + class Mutex: """File and pthread lock based resource mutex""" - def __init__(self, lock_file): - self.thread_mutex = threading.Lock() - self.lock_path = '/var/lock/' + lock_file + def __init__(self, lock_name): + if not lock_name_pattern.match(lock_name): + raise ValueError('Bad lock name') + self._lock_path = '/var/lock/plinth-%s.lock' % lock_name + self._thread_mutex = threading.Lock() + + @property + def lock_path(self): + return self._lock_path @contextlib.contextmanager def lock_threads_only(self): """Acquire the thread lock but not the file lock""" - if not self.thread_mutex.acquire(timeout=5): + if not self._thread_mutex.acquire(timeout=5): raise RuntimeError('Could not acquire thread lock') try: yield finally: - self.thread_mutex.release() + self._thread_mutex.release() @contextlib.contextmanager def lock_all(self): @@ -45,36 +56,48 @@ class Mutex: def _open_lock_file(self): """Attempt to open lock file for R&W. Raises OSError on failure""" - if os.getuid() == 0: - if os.path.exists(self.lock_path): - # Check its owner and mode - stat = os.stat(self.lock_path) - owner = pwd.getpwuid(stat.st_uid).pw_name - mode = stat.st_mode & 0o777 - if owner != 'plinth' or mode != 0o600: - logger.warning('Clean up bad file %s', self.lock_path) - os.unlink(self.lock_path) - self._create_lock_file_as_plinth() - else: - self._create_lock_file_as_plinth() + og_ruid, og_euid, og_suid = os.getresuid() + if og_euid == 0 and threading.active_count() > 1: + raise RaceCondition('setuid in a multi-threaded process') + if not os.path.exists(self.lock_path): + self._create_lock_file_as_plinth() - return open(self.lock_path, 'wb+') + fd = None + try: + if og_euid == 0: + # Temporarily run the current process as plinth + plinth_uid = pwd.getpwnam('plinth').pw_uid + self._checked_setresuid(og_ruid, plinth_uid, 0) + fd = open(self.lock_path, 'w+b') + finally: + # Restore resuid + if og_euid == 0: + self._checked_setresuid(og_ruid, 0, 0) + if og_suid != 0: + self._checked_setresuid(og_ruid, 0, og_suid) + + return fd def _create_lock_file_as_plinth(self): - args = ['sudo', '-n', '-u', 'plinth', 'touch', self.lock_path] + # Don't change the current processes umask + # Do create a new process + args = ['sudo', '-n', '-u', 'plinth', '/bin/sh', '-c'] + args.append('umask 177 && > ' + self.lock_path) + completed = subprocess.run(args, capture_output=True) if completed.returncode != 0: - logger.critical('Process returned %d', completed.returncode) + logger.critical('Subprocess returned %d', completed.returncode) logger.critical('Stdout: %r', completed.stdout) logger.critical('Stderr: %r', completed.stderr) raise OSError('Could not create ' + self.lock_path) - os.chmod(self.lock_path, 0o600) - def _try(self, function): - try: - return 0, function() - except OSError as error: - if error.errno in (errno.EACCES, errno.EPERM): - return error.errno, None - else: - raise + def _checked_setresuid(self, ruid, euid, suid): + os.setresuid(ruid, euid, suid) + if os.getresuid() != (ruid, euid, suid): + try: + raise SystemExit('PANIC: setresuid failed') + except SystemExit as e: + # Print stack trace + logger.exception(e) + # Force exit + exit(1) diff --git a/plinth/modules/email_server/postconf.py b/plinth/modules/email_server/postconf.py index 2553f24d3..caf443ea5 100644 --- a/plinth/modules/email_server/postconf.py +++ b/plinth/modules/email_server/postconf.py @@ -6,7 +6,7 @@ import re import subprocess from .lock import Mutex -postconf_mutex = Mutex('plinth-email-postconf.lock') +postconf_mutex = Mutex('email-postconf') @dataclasses.dataclass From 0acbe5dd6b23f28346f5081ce57ef0f00211e187 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Wed, 7 Jul 2021 22:05:41 +0000 Subject: [PATCH 20/56] email: Support UID number lookup in Dovecot --- .../data/etc/dovecot/conf.d/05-freedombox-auth.conf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plinth/modules/email_server/data/etc/dovecot/conf.d/05-freedombox-auth.conf b/plinth/modules/email_server/data/etc/dovecot/conf.d/05-freedombox-auth.conf index b8b027b42..4d3cb8e22 100644 --- a/plinth/modules/email_server/data/etc/dovecot/conf.d/05-freedombox-auth.conf +++ b/plinth/modules/email_server/data/etc/dovecot/conf.d/05-freedombox-auth.conf @@ -12,6 +12,15 @@ passdb { result_success = return-ok } +userdb { + # UID number lookup (10001@example.com) + driver = ldap + args = /etc/dovecot/freedombox-ldap-userdb-aliases.conf.ext + result_failure = continue + result_internalfail = return-fail + result_success = return-ok +} + userdb { driver = ldap args = /etc/dovecot/freedombox-ldap-userdb.conf.ext From 8c740e08dad4920ed91ef36b36916789e805b290 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Thu, 8 Jul 2021 07:54:28 +0000 Subject: [PATCH 21/56] email: diagnostics: Fix sudo permission problem --- plinth/modules/email_server/lock.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plinth/modules/email_server/lock.py b/plinth/modules/email_server/lock.py index f613db75e..d658c261d 100644 --- a/plinth/modules/email_server/lock.py +++ b/plinth/modules/email_server/lock.py @@ -60,7 +60,7 @@ class Mutex: if og_euid == 0 and threading.active_count() > 1: raise RaceCondition('setuid in a multi-threaded process') if not os.path.exists(self.lock_path): - self._create_lock_file_as_plinth() + self._create_lock_file_as_plinth(og_euid) fd = None try: @@ -78,10 +78,13 @@ class Mutex: return fd - def _create_lock_file_as_plinth(self): + def _create_lock_file_as_plinth(self, your_euid): # Don't change the current processes umask # Do create a new process - args = ['sudo', '-n', '-u', 'plinth', '/bin/sh', '-c'] + args = [] + if your_euid == 0: + args.extend(['sudo', '-n', '-u', 'plinth']) + args.extend(['/bin/sh', '-c']) args.append('umask 177 && > ' + self.lock_path) completed = subprocess.run(args, capture_output=True) From a9ac51eb7baf2fd9f9baf3ca2fe1fe4176a840e1 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Thu, 8 Jul 2021 08:34:51 +0000 Subject: [PATCH 22/56] email: views: Implement tab rendering --- .../email_server/templates/email_server.html | 13 ++++++ plinth/modules/email_server/views.py | 43 ++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 plinth/modules/email_server/templates/email_server.html diff --git a/plinth/modules/email_server/templates/email_server.html b/plinth/modules/email_server/templates/email_server.html new file mode 100644 index 000000000..cbf662eb1 --- /dev/null +++ b/plinth/modules/email_server/templates/email_server.html @@ -0,0 +1,13 @@ +{% extends "app.html" %} + +{% load i18n %} + +{% block configuration %} + {{ tabs|safe }} +

+ + {% trans "Visit Rspamd administration interface" %} + +

+ {{ block.super }} +{% endblock %} diff --git a/plinth/modules/email_server/views.py b/plinth/modules/email_server/views.py index 615142512..e27160939 100644 --- a/plinth/modules/email_server/views.py +++ b/plinth/modules/email_server/views.py @@ -1,15 +1,56 @@ # SPDX-License-Identifier: AGPL-3.0-or-later -import plinth +import io + +import plinth.views +from django.shortcuts import render +from django.utils.html import escape +from django.utils.translation import ugettext_lazy as _ + from . import forms +tabs = [ + ('', 'Home'), + ('alias', 'Alias'), + ('relay', 'Relay'), + ('security', 'Security') +] + class EmailServerView(plinth.views.AppView): """Server configuration page""" app_id = 'email_server' form_class = forms.EmailServerForm + template_name = 'email_server.html' def form_valid(self, form): # old_settings = form.initial # new_status = form.cleaned_data # plinth.actions.superuser_run('email_server', ['--help']) return super().form_valid(form) + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context['tabs'] = render_tabs(self.request) + return context + + +def render_tabs(request): + sb = io.StringIO() + sb.write('') + return sb.getvalue() From 4375828703ec5acd3dcb2e5708c551349bcc1221 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Wed, 14 Jul 2021 00:27:07 +0000 Subject: [PATCH 23/56] email: Implement alias management - Separate alias database from system - Block mail to system users, without backscatter - Alias management UI for non-admin users - Enabling/Disabling aliases (mails to /dev/null) Misc. changes - Daemon management - Backup information - Postconf diagnostics interface --- plinth/modules/email_server/__init__.py | 12 +- .../modules/email_server/aliases/__init__.py | 151 ++++++++++++++++++ plinth/modules/email_server/aliases/models.py | 31 ++++ plinth/modules/email_server/audit/ldap.py | 125 ++++++++++++++- plinth/modules/email_server/audit/models.py | 24 ++- plinth/modules/email_server/audit/spam.py | 4 +- .../freedombox-username-to-uid-number.cf | 9 ++ plinth/modules/email_server/forms.py | 12 +- plinth/modules/email_server/manifest.py | 11 ++ plinth/modules/email_server/postconf.py | 27 +++- .../modules/email_server/templates/alias.html | 52 ++++++ plinth/modules/email_server/urls.py | 5 +- plinth/modules/email_server/views.py | 137 +++++++++++++++- 13 files changed, 570 insertions(+), 30 deletions(-) create mode 100644 plinth/modules/email_server/aliases/__init__.py create mode 100644 plinth/modules/email_server/aliases/models.py create mode 100644 plinth/modules/email_server/data/etc/postfix/freedombox-username-to-uid-number.cf create mode 100644 plinth/modules/email_server/templates/alias.html diff --git a/plinth/modules/email_server/__init__.py b/plinth/modules/email_server/__init__.py index c8d6faff1..15fe9df30 100644 --- a/plinth/modules/email_server/__init__.py +++ b/plinth/modules/email_server/__init__.py @@ -17,9 +17,10 @@ from . import manifest version = 1 managed_packages = ['postfix', 'dovecot-pop3d', 'dovecot-imapd', - 'dovecot-lmtpd', 'dovecot-ldap', 'dovecot-managesieved', - 'rspamd'] -managed_services = ['postfix', 'dovecot'] + 'dovecot-ldap', 'dovecot-lmtpd', 'dovecot-managesieved', + 'rspamd', 'clamav', 'clamav-daemon'] +managed_services = ['postfix', 'dovecot', 'rspamd', 'redis', 'clamav-daemon', + 'clamav-freshclam'] app = None @@ -62,7 +63,7 @@ class EmailServerApp(plinth.app.App): name=info.name, short_description=info.short_description, icon='roundcube', - configure_url=reverse_lazy('email_server'), + url=reverse_lazy('email_server:my_aliases'), clients=manifest.clients, login_required=True ) @@ -85,6 +86,9 @@ class EmailServerApp(plinth.app.App): listen_ports=dovecot_ports) self.add(postfixd) self.add(dovecotd) + for name in ('rspamd', 'redis', 'clamav-daemon', 'clamav-freshclam'): + daemon = plinth.daemon.Daemon('daemon-' + name, name) + self.add(daemon) # Ports firewall = Firewall('firewall-email', info.name, diff --git a/plinth/modules/email_server/aliases/__init__.py b/plinth/modules/email_server/aliases/__init__.py new file mode 100644 index 000000000..e510dd3c3 --- /dev/null +++ b/plinth/modules/email_server/aliases/__init__.py @@ -0,0 +1,151 @@ +"""Manages email aliases""" +# SPDX-License-Identifier: AGPL-3.0-or-later + +import contextlib +import dbm +import logging +import os +import pwd +import sqlite3 + +from django.core.exceptions import ValidationError +from django.utils.translation import ugettext_lazy as _ + +from plinth.modules.email_server import lock +from . import models + +map_db_schema_script = """ +PRAGMA journal_mode=WAL; +BEGIN; +CREATE TABLE IF NOT EXISTS Alias ( + email_name TEXT NOT NULL, + uid_number INTEGER NOT NULL, + status INTEGER NOT NULL, + PRIMARY KEY (email_name) +); +COMMIT; +""" + +mailsrv_dir = '/var/lib/plinth/mailsrv' +hash_db_path = mailsrv_dir + '/aliases' +sqlite_db_path = mailsrv_dir + '/aliases.sqlite3' + +alias_sync_mutex = lock.Mutex('alias-sync') +logger = logging.getLogger(__name__) + + +@contextlib.contextmanager +def db_cursor(): + # Turn ON autocommit mode + con = sqlite3.connect(sqlite_db_path, isolation_level=None) + con.row_factory = sqlite3.Row + try: + cur = con.cursor() + yield cur + finally: + con.close() + + +def get(uid_number): + s = 'SELECT * FROM Alias WHERE uid_number=?' + with db_cursor() as cur: + rows = cur.execute(s, (uid_number,)) + result = [models.Alias(**r) for r in rows] + return result + + +def put(uid_number, email_name): + s = """INSERT INTO Alias(email_name, uid_number, status) + SELECT ?,?,? WHERE NOT EXISTS( + SELECT 1 FROM Alias WHERE email_name=? + )""" + email_name = models.sanitize_email_name(email_name) + # email_name cannot be the same as a user name + try: + pwd.getpwnam(email_name) + raise ValidationError(_('The alias was taken')) + except KeyError: + pass + + with db_cursor() as cur: + cur.execute(s, (email_name, uid_number, 1, email_name)) + if cur.rowcount == 0: + raise ValidationError(_('The alias was taken')) + + schedule_hash_update() + + +def delete(uid_number, alias_list): + s = 'DELETE FROM Alias WHERE uid_number=? AND email_name=?' + for i in range(len(alias_list)): + alias_list[i] = models.sanitize_email_name(alias_list[i]) + + parameter_seq = ((uid_number, a) for a in alias_list) + with db_cursor() as cur: + cur.execute('BEGIN') + cur.executemany(s, parameter_seq) + cur.execute('COMMIT') + schedule_hash_update() + + +def set_enabled(uid_number, alias_list): + return _set_status(uid_number, alias_list, 1) + + +def set_disabled(uid_number, alias_list): + return _set_status(uid_number, alias_list, 0) + + +def _set_status(uid_number, alias_list, status): + s = 'UPDATE Alias SET status=? WHERE uid_number=? AND email_name=?' + for i in range(len(alias_list)): + alias_list[i] = models.sanitize_email_name(alias_list[i]) + + parameter_seq = ((status, uid_number, a) for a in alias_list) + with db_cursor() as cur: + cur.execute('BEGIN') + cur.executemany(s, parameter_seq) + cur.execute('COMMIT') + schedule_hash_update() + + +def schedule_hash_update(): + tmp = hash_db_path + '-tmp' + with alias_sync_mutex.lock_all(), db_cursor() as cur: + all_aliases = cur.execute('SELECT * FROM Alias') + + # Delete the temp file if exists + if os.path.exists(tmp): + os.unlink(tmp) + + # Create new alias db at temp path + db = dbm.ndbm.open(tmp, 'c') + try: + for row in all_aliases: + alias = models.Alias(**row) + key = alias.email_name.encode('ascii') + b'\0' + if alias.enabled: + value = str(alias.uid_number).encode('ascii') + b'\0' + else: + value = b'/dev/null\0' + db[key] = value + finally: + db.close() + + # Atomically replace old alias db, rename(2) + os.rename(tmp + '.db', hash_db_path + '.db') + + +def first_setup(): + _create_db_schema_if_not_exists() + schedule_hash_update() + + +def _create_db_schema_if_not_exists(): + # Create folder + if not os.path.isdir(mailsrv_dir): + os.mkdir(mailsrv_dir) + # Create schema if not exists + with db_cursor() as cur: + cur.executescript(map_db_schema_script) + diff --git a/plinth/modules/email_server/aliases/models.py b/plinth/modules/email_server/aliases/models.py new file mode 100644 index 000000000..ac3277fe1 --- /dev/null +++ b/plinth/modules/email_server/aliases/models.py @@ -0,0 +1,31 @@ +import re +from dataclasses import dataclass, field, InitVar + +from django.core.exceptions import ValidationError +from django.utils.translation import ugettext_lazy as _ + +email_positive_pattern = re.compile('^[a-zA-Z0-9-_\\.]+') + + +def sanitize_email_name(email_name): + email_name = email_name.strip().lower() + if len(email_name) < 2: + raise ValidationError(_('Must be at least 2 characters long')) + if not re.match('^[a-z0-9-_\\.]+$', email_name): + raise ValidationError(_('Contains illegal characters')) + if not re.match('^[a-z0-9].*[a-z0-9]$', email_name): + raise ValidationError(_('Must start and end with a-z or 0-9')) + if re.match('^[0-9]+$', email_name): + raise ValidationError(_('Cannot be a number')) + return email_name + + +@dataclass +class Alias: + uid_number: int + email_name: str + enabled: bool = field(init=False) + status: InitVar[int] + + def __post_init__(self, status): + self.enabled = (status != 0) diff --git a/plinth/modules/email_server/audit/ldap.py b/plinth/modules/email_server/audit/ldap.py index 790b8a953..f0af7a7bd 100644 --- a/plinth/modules/email_server/audit/ldap.py +++ b/plinth/modules/email_server/audit/ldap.py @@ -1,4 +1,5 @@ -"""Audit of LDAP and mail submission settings""" +"""Provides the diagnosis of SASL, mail submission, and user database lookup +configurations""" # SPDX-License-Identifier: AGPL-3.0-or-later import logging @@ -6,6 +7,7 @@ import logging from plinth import actions import plinth.modules.email_server.postconf as postconf +import plinth.modules.email_server.aliases as aliases from . import models default_config = { @@ -40,6 +42,12 @@ default_smtps_options = { 'smtpd_relay_restrictions': 'permit_sasl_authenticated,reject' } + +MAILSRV_DIR = '/var/lib/plinth/mailsrv' +ETC_ALIASES = 'hash:/etc/aliases' +BEFORE_ALIASES = 'ldap:/etc/postfix/freedombox-username-to-uid-number.cf' +AFTER_ALIASES = 'hash:' + aliases.hash_db_path + logger = logging.getLogger(__name__) @@ -50,19 +58,27 @@ def get(): GET /audit/ldap """ results = [] - with postconf.postconf_mutex.lock_all(): + with postconf.mutex.lock_all(): results.append(check_sasl()) + results.append(check_alias_maps()) + results.append(check_local_recipient_maps()) return results def repair(): - """Tries to repair LDAP and mail submission settings + """Tries to repair SASL, mail submission, and user lookup settings Recommended endpoint name: POST /audit/ldap/repair """ - actions.superuser_run('email_server', ['-i', 'ldap', 'set_sasl']) - actions.superuser_run('email_server', ['-i', 'ldap', 'set_submission']) + aliases.first_setup() + actions.superuser_run('email_server', ['-i', 'ldap', 'set_up']) + + +def action_set_up(): + action_set_sasl() + action_set_submission() + action_set_ulookup() def check_sasl(): @@ -79,13 +95,13 @@ def fix_sasl(diagnosis): def action_set_sasl(): - """Called by email_server ipc ldap set_sasl""" - with postconf.postconf_mutex.lock_all(): + """Handles email_server -i ldap set_sasl""" + with postconf.mutex.lock_all(): fix_sasl(check_sasl()) def action_set_submission(): - """Called by email_server ipc set_submission""" + """Handles email_server -i ldap set_submission""" logger.info('Set postfix service: %r', default_submission_options) postconf.set_master_cf_options(service_flags=submission_flags, options=default_submission_options) @@ -93,3 +109,96 @@ def action_set_submission(): logger.info('Set postfix service: %r', default_smtps_options) postconf.set_master_cf_options(service_flags=smtps_flags, options=default_smtps_options) + + +def check_alias_maps(): + """Check the ability to mail to usernames and user aliases""" + diagnosis = models.MainCfDiagnosis('Postfix alias maps') + + analysis = models.AliasMapsAnalysis() + analysis.parsed = postconf.parse_maps_by_key_unsafe('alias_maps') + analysis.isystem = list_find(analysis.parsed, ETC_ALIASES) + analysis.ibefore = list_find(analysis.parsed, BEFORE_ALIASES) + analysis.iafter = list_find(analysis.parsed, AFTER_ALIASES) + + if analysis.ibefore == -1 or analysis.iafter == -1: + diagnosis.flag_once('alias_maps', user=analysis) + diagnosis.critical('Required maps not in list') + if analysis.ibefore > analysis.iafter: + diagnosis.flag_once('alias_maps', user=analysis) + diagnosis.critical('Insecure map order') + + return diagnosis + + +def fix_alias_maps(diagnosis): + unresolved_issues = list(diagnosis.unresolved_issues()) + if 'alias_maps' in unresolved_issues: + analysis = diagnosis.user['alias_maps'] + # Delete *all* references to BEFORE_ALIASES and AFTER_ALIASES + for i in range(len(analysis.parsed)): + if analysis.parsed[i] in (BEFORE_ALIASES, AFTER_ALIASES): + analysis.parsed[i] = '' + # Does hash:/etc/aliases exist? + if analysis.isystem >= 0: + # Put the maps around hash:/etc/aliases + val = '%s %s %s' % (BEFORE_ALIASES, ETC_ALIASES, AFTER_ALIASES) + analysis.parsed[analysis.isystem] = val + else: + # To the end + analysis.parsed.append(BEFORE_ALIASES) + analysis.parsed.append(AFTER_ALIASES) + # List -> string + fixed = ' '.join(filter(None, analysis.parsed)) + diagnosis.advice['alias_maps'] = fixed + + diagnosis.assert_resolved() + logging.info('Setting postfix config: %r', diagnosis.advice) + postconf.set_many_unsafe(diagnosis.advice) + + +def check_local_recipient_maps(): + diagnosis = models.MainCfDiagnosis('Postfix local recipient maps') + lrcpt_maps = postconf.parse_maps_by_key_unsafe('local_recipient_maps') + list_modified = False + + # Block mails to system users + # local_recipient_maps must not contain proxy:unix:passwd.byname + ipasswd = list_find(lrcpt_maps, 'proxy:unix:passwd.byname') + if ipasswd >= 0: + diagnosis.critical('Mail to system users (/etc/passwd) possible') + # Propose a fix + lrcpt_maps[ipasswd] = '' + list_modified = True + + if list_modified: + fix = ' '.join(filter(None, lrcpt_maps)) + diagnosis.flag('local_recipient_maps', corrected_value=fix) + + return diagnosis + + +def fix_local_recipient_maps(diagnosis): + diagnosis.assert_resolved() + logging.info('Setting postfix config: %r', diagnosis.advice) + postconf.set_many_unsafe(diagnosis.advice) + + +def action_set_ulookup(): + """Handles email_server -i ldap set_ulookup""" + with postconf.mutex.lock_all(): + fix_alias_maps(check_alias_maps()) + fix_local_recipient_maps(check_local_recipient_maps()) + + +def list_find(lst, element, start=None, end=None): + if start is None: + start = 0 + if end is None: + end = len(lst) + if start < 0 or end < 0: + return -1 + try: + return lst.index(element, start, end) + except ValueError: + return -1 diff --git a/plinth/modules/email_server/audit/models.py b/plinth/modules/email_server/audit/models.py index a08efb0a2..9debf504e 100644 --- a/plinth/modules/email_server/audit/models.py +++ b/plinth/modules/email_server/audit/models.py @@ -1,6 +1,8 @@ """Audit models""" # SPDX-License-Identifier: AGPL-3.0-or-later +import dataclasses import logging +import typing logger = logging.getLogger(__name__) @@ -57,8 +59,15 @@ class MainCfDiagnosis(Diagnosis): self.user = {} def flag(self, key, corrected_value=None, user=None): - self.advice[key] = corrected_value - self.user[key] = user + if key in self.advice: + raise ValueError('Key has been flagged') + else: + self.advice[key] = corrected_value + self.user[key] = user + + def flag_once(self, key, **kwargs): + if key not in self.advice: + self.flag(key, **kwargs) def unresolved_issues(self): """Returns an interator of dictionary keys""" @@ -79,3 +88,14 @@ class MainCfDiagnosis(Diagnosis): unresolved issue""" if None in self.advice.values(): raise UnresolvedIssueError('Assertion failed') + + +@dataclasses.dataclass(init=False) +class AliasMapsAnalysis: + parsed = typing.List[str] + ibefore = int + isystem = int + iafter = int + + def __init__(self): + pass diff --git a/plinth/modules/email_server/audit/spam.py b/plinth/modules/email_server/audit/spam.py index f0ad26fe3..8a8f0c24b 100644 --- a/plinth/modules/email_server/audit/spam.py +++ b/plinth/modules/email_server/audit/spam.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) def get(): results = [] - with postconf.postconf_mutex.lock_all(): + with postconf.mutex.lock_all(): results.append(check_filter()) return results @@ -48,5 +48,5 @@ def fix_filter(diagnosis): def action_set_filter(): - with postconf.postconf_mutex.lock_all(): + with postconf.mutex.lock_all(): fix_filter(check_filter()) diff --git a/plinth/modules/email_server/data/etc/postfix/freedombox-username-to-uid-number.cf b/plinth/modules/email_server/data/etc/postfix/freedombox-username-to-uid-number.cf new file mode 100644 index 000000000..c8919becf --- /dev/null +++ b/plinth/modules/email_server/data/etc/postfix/freedombox-username-to-uid-number.cf @@ -0,0 +1,9 @@ +# This file is managed by FreedomBox + +# Map user name to UID number + +bind = no +server_host = ldap://127.0.0.1 +search_base = dc=thisbox +query_filter = (&(objectClass=posixAccount)(uid=%s)) +result_attribute = uidNumber diff --git a/plinth/modules/email_server/forms.py b/plinth/modules/email_server/forms.py index 4f6ecdac7..a38123a8a 100644 --- a/plinth/modules/email_server/forms.py +++ b/plinth/modules/email_server/forms.py @@ -1,6 +1,14 @@ # SPDX-License-Identifier: AGPL-3.0-or-later from django import forms - +from django.utils.translation import ugettext_lazy as _ class EmailServerForm(forms.Form): - pass + domain = forms.CharField(label=_('domain'), max_length=256) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class AliasCreationForm(forms.Form): + email_name = forms.CharField(label=_('New alias (without @domain)'), + max_length=50) diff --git a/plinth/modules/email_server/manifest.py b/plinth/modules/email_server/manifest.py index 3a6bdae77..6426b50fe 100644 --- a/plinth/modules/email_server/manifest.py +++ b/plinth/modules/email_server/manifest.py @@ -50,3 +50,14 @@ clients = [{ 'url': store_url('google-play', 'eu.faircode.email') }] }] + +backup = { + 'data': { + 'directories': [ + '/var/lib/plinth/mailsrv', + '/etc/postfix', + '/etc/dovecot' + ] + }, + 'services': ['postfix', 'dovecot'] +} diff --git a/plinth/modules/email_server/postconf.py b/plinth/modules/email_server/postconf.py index caf443ea5..749e6f99d 100644 --- a/plinth/modules/email_server/postconf.py +++ b/plinth/modules/email_server/postconf.py @@ -6,7 +6,7 @@ import re import subprocess from .lock import Mutex -postconf_mutex = Mutex('email-postconf') +mutex = Mutex('email-postconf') @dataclasses.dataclass @@ -31,7 +31,7 @@ def get_many(key_list): Return a key-value map""" for key in key_list: validate_key(key) - with postconf_mutex.lock_all(): + with mutex.lock_all(): return get_many_unsafe(key_list) @@ -48,7 +48,7 @@ def set_many(kv_map): validate_key(key) validate_value(value) - with postconf_mutex.lock_all(): + with mutex.lock_all(): set_many_unsafe(kv_map) @@ -57,7 +57,7 @@ def set_many_unsafe(kv_map): set_unsafe(key, value) -def set_master_cf_options(service_flags, options): +def set_master_cf_options(service_flags, options={}): """Acquire resource lock. Set master.cf service options""" if not isinstance(service_flags, ServiceFlags): raise TypeError('service_flags') @@ -68,7 +68,7 @@ def set_master_cf_options(service_flags, options): service_slash_type = service_flags.service + '/' + service_flags.type flag_string = service_flags.serialize() - with postconf_mutex.lock_all(): + with mutex.lock_all(): # /sbin/postconf -M "service/type=flag_string" set_unsafe(service_slash_type, flag_string, '-M') for short_key, value in options.items(): @@ -93,6 +93,23 @@ def set_unsafe(key, value, flag=''): _run(['/sbin/postconf', '{}={}'.format(key, value)]) +def parse_maps(raw_value): + if '{' in raw_value or '}' in raw_value: + raise ValueError('Unsupported map list format') + + value_list = [] + for segment in raw_value.split(','): + for sub_segment in segment.strip().split(' '): + sub_segment = sub_segment.strip() + if sub_segment: + value_list.append(sub_segment) + return value_list + + +def parse_maps_by_key_unsafe(key): + return parse_maps(get_unsafe(key)) + + def _run(args): """Run process. Capture and return standard output as a string. Raise a RuntimeError on non-zero exit codes""" diff --git a/plinth/modules/email_server/templates/alias.html b/plinth/modules/email_server/templates/alias.html new file mode 100644 index 000000000..e8b072d49 --- /dev/null +++ b/plinth/modules/email_server/templates/alias.html @@ -0,0 +1,52 @@ +{% extends "app.html" %} + +{% load bootstrap %} +{% load i18n %} + +{% block configuration %} + + {{ tabs|safe }} +

{% trans "Alias Management" %}

+ + {% if error %} + + {% endif %} + + {% if no_alias %} +

{% trans "You have no email aliases." %}

+ {% else %} +
+ {% csrf_token %} + {{ alias_boxes|safe }} + + + + + + + + +
+ {% endif %} + +

{% trans "Create a new email alias" %}

+ +
+ {% csrf_token %} + {{ form|bootstrap }} + + +
+ +{% endblock %} diff --git a/plinth/modules/email_server/urls.py b/plinth/modules/email_server/urls.py index 61d5844af..eae2a2d7c 100644 --- a/plinth/modules/email_server/urls.py +++ b/plinth/modules/email_server/urls.py @@ -1,8 +1,11 @@ # SPDX-License-Identifier: AGPL-3.0-or-later from django.urls import path +from plinth.utils import non_admin_view from . import views urlpatterns = [ - path('apps/email_server/', views.EmailServerView.as_view(), name='index') + path('apps/email_server/', views.EmailServerView.as_view(), name='index'), + path('apps/email_server/my_aliases', + non_admin_view(views.AliasView.as_view()), name='my_aliases') ] diff --git a/plinth/modules/email_server/views.py b/plinth/modules/email_server/views.py index e27160939..ec8663a7b 100644 --- a/plinth/modules/email_server/views.py +++ b/plinth/modules/email_server/views.py @@ -1,18 +1,23 @@ # SPDX-License-Identifier: AGPL-3.0-or-later import io +import itertools +import pwd import plinth.views -from django.shortcuts import render + +from django.core.exceptions import ValidationError from django.utils.html import escape from django.utils.translation import ugettext_lazy as _ +from django.views.generic.base import TemplateView from . import forms +from . import aliases tabs = [ - ('', 'Home'), - ('alias', 'Alias'), - ('relay', 'Relay'), - ('security', 'Security') + ('', _('Home')), + ('alias', _('Alias')), + ('relay', _('Relay')), + ('security', _('Security')) ] @@ -34,6 +39,126 @@ class EmailServerView(plinth.views.AppView): return context +class AliasView(TemplateView): + class Checkboxes: + def __init__(self, post=None, initial=None): + self.models = initial + self.post = post + self.cleaned_data = {} + + def render(self): + if self.models is None: + raise RuntimeError('Uninitialized form') + sb = io.StringIO() + enabled = [a.email_name for a in self.models if a.enabled] + disabled = [a.email_name for a in self.models if not a.enabled] + + if len(enabled) > 0: + sb.write('
') + sb.write('%s' % escape(_('Enabled'))) + self._render_boxes(enabled, 'enabled', sb) + sb.write('
') + if len(disabled) > 0: + sb.write('
') + sb.write('%s' % escape(_('Disabled'))) + self._render_boxes(disabled, 'disabled', sb) + sb.write('
') + return sb.getvalue() + + @staticmethod + def _render_boxes(email_names, suffix, sb): + for i, email_name in enumerate(email_names): + input_id = 'cb_alias_%s_%d' % (suffix, i) + value = escape(email_name) + sb.write('
') + sb.write('' % (input_id, value)) + sb.write('' % (input_id, value)) + sb.write('
') + + def is_valid(self): + lst = list(filter(None, self.post.getlist('alias'))) + if not lst: + return False + else: + self.cleaned_data['alias'] = lst + return True + + template_name = 'alias.html' + form_classes = (forms.AliasCreationForm, Checkboxes) + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context['form'] = forms.AliasCreationForm() + + uid = pwd.getpwnam(self.request.user.username).pw_uid + models = aliases.get(uid) + if len(models) > 0: + form = AliasView.Checkboxes(initial=models) + context['alias_boxes'] = form.render() + else: + context['no_alias'] = True + return context + + def find_form(self, post): + form_name = post.get('form') + for cls in self.form_classes: + if cls.__name__ == form_name: + return cls(post) + raise ValidationError('Form was unspecified') + + def find_button(self, post): + key_filter = (k for k in post.keys() if k.startswith('btn_')) + lst = list(itertools.islice(key_filter, 2)) + if len(lst) != 1: + raise ValidationError('Bad post data') + if not isinstance(lst[0], str): + raise ValidationError('Bad post data') + return lst[0][len('btn_'):] + + def post(self, request): + try: + return self._post(request) + except ValidationError as e: + context = self.get_context_data() + context['error'] = e + return self.render_to_response(context, status=400) + + def _post(self, request): + form = self.find_form(request.POST) + button = self.find_button(request.POST) + if not form.is_valid(): + raise ValidationError('Form invalid') + + if isinstance(form, AliasView.Checkboxes): + if button not in ('delete', 'disable', 'enable'): + raise ValidationError('Bad button') + return self.alias_operation_form_valid(form, button) + + if isinstance(form, forms.AliasCreationForm): + if button != 'add': + raise ValidationError('Bad button') + return self.alias_creation_form_valid(form, button) + + raise RuntimeError('Unknown form') + + def alias_operation_form_valid(self, form, button): + uid = pwd.getpwnam(self.request.user.username).pw_uid + alias_list = form.cleaned_data['alias'] + if button == 'delete': + aliases.delete(uid, alias_list) + elif button == 'disable': + aliases.set_disabled(uid, alias_list) + elif button == 'enable': + aliases.set_enabled(uid, alias_list) + return self.render_to_response(self.get_context_data()) + + def alias_creation_form_valid(self, form, button): + uid = pwd.getpwnam(self.request.user.username).pw_uid + aliases.put(uid, form.cleaned_data['email_name']) + return self.render_to_response(self.get_context_data()) + + def render_tabs(request): sb = io.StringIO() sb.write('') From 2bd1ad45334ddce1ae14ea83085d35405788d4ca Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Sat, 17 Jul 2021 06:21:30 +0000 Subject: [PATCH 24/56] email: aliases: Use bootstrap styles Other changes: - Added license identifiers - Fixed linter warnings - Deleted excessive parameters in rendering code --- .../modules/email_server/aliases/__init__.py | 1 - plinth/modules/email_server/forms.py | 1 + .../modules/email_server/templates/alias.html | 1 + .../email_server/templates/email_server.html | 1 + plinth/modules/email_server/views.py | 55 ++++++++++++------- 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/plinth/modules/email_server/aliases/__init__.py b/plinth/modules/email_server/aliases/__init__.py index e510dd3c3..82de702b5 100644 --- a/plinth/modules/email_server/aliases/__init__.py +++ b/plinth/modules/email_server/aliases/__init__.py @@ -148,4 +148,3 @@ def _create_db_schema_if_not_exists(): # Create schema if not exists with db_cursor() as cur: cur.executescript(map_db_schema_script) - diff --git a/plinth/modules/email_server/forms.py b/plinth/modules/email_server/forms.py index a38123a8a..9567d1be1 100644 --- a/plinth/modules/email_server/forms.py +++ b/plinth/modules/email_server/forms.py @@ -2,6 +2,7 @@ from django import forms from django.utils.translation import ugettext_lazy as _ + class EmailServerForm(forms.Form): domain = forms.CharField(label=_('domain'), max_length=256) diff --git a/plinth/modules/email_server/templates/alias.html b/plinth/modules/email_server/templates/alias.html index e8b072d49..cab40ab5c 100644 --- a/plinth/modules/email_server/templates/alias.html +++ b/plinth/modules/email_server/templates/alias.html @@ -1,3 +1,4 @@ +{# SPDX-License-Identifier: AGPL-3.0-or-later #} {% extends "app.html" %} {% load bootstrap %} diff --git a/plinth/modules/email_server/templates/email_server.html b/plinth/modules/email_server/templates/email_server.html index cbf662eb1..89557bdb5 100644 --- a/plinth/modules/email_server/templates/email_server.html +++ b/plinth/modules/email_server/templates/email_server.html @@ -1,3 +1,4 @@ +{# SPDX-License-Identifier: AGPL-3.0-or-later #} {% extends "app.html" %} {% load i18n %} diff --git a/plinth/modules/email_server/views.py b/plinth/modules/email_server/views.py index ec8663a7b..f940a3e21 100644 --- a/plinth/modules/email_server/views.py +++ b/plinth/modules/email_server/views.py @@ -45,36 +45,49 @@ class AliasView(TemplateView): self.models = initial self.post = post self.cleaned_data = {} + # HTML rendering + self.sb = io.StringIO() + self.counter = 0 def render(self): if self.models is None: raise RuntimeError('Uninitialized form') - sb = io.StringIO() + if self.sb.tell() > 0: + raise RuntimeError('render has been called') + enabled = [a.email_name for a in self.models if a.enabled] disabled = [a.email_name for a in self.models if not a.enabled] - if len(enabled) > 0: - sb.write('
') - sb.write('%s' % escape(_('Enabled'))) - self._render_boxes(enabled, 'enabled', sb) - sb.write('
') - if len(disabled) > 0: - sb.write('
') - sb.write('%s' % escape(_('Disabled'))) - self._render_boxes(disabled, 'disabled', sb) - sb.write('
') - return sb.getvalue() + self._render_fieldset(enabled, _('Enabled aliases')) + self._render_fieldset(disabled, _('Disabled aliases')) - @staticmethod - def _render_boxes(email_names, suffix, sb): - for i, email_name in enumerate(email_names): - input_id = 'cb_alias_%s_%d' % (suffix, i) + return self.sb.getvalue() + + def _render_fieldset(self, email_names, legend): + if len(email_names) > 0: + self.sb.write('
') + self.sb.write('%s' % escape(legend)) + self._render_boxes(email_names) + self.sb.write('
') + + def _render_boxes(self, email_names): + for email_name in email_names: + input_id = 'cb_alias_%d' % self._count() value = escape(email_name) - sb.write('
') - sb.write('' % (input_id, value)) - sb.write('' % (input_id, value)) - sb.write('
') + self.sb.write('
') + + self.sb.write('' % (input_id, value)) + + self.sb.write('' % (input_id, value)) + + self.sb.write('
') + + def _count(self): + self.counter += 1 + return self.counter def is_valid(self): lst = list(filter(None, self.post.getlist('alias'))) From 62c501e9c7d2849c80761f61871f21d7e9246414 Mon Sep 17 00:00:00 2001 From: fliu <10025-fliu@users.noreply.salsa.debian.org> Date: Tue, 20 Jul 2021 05:28:11 +0000 Subject: [PATCH 25/56] email: Add UI for creating the home directory email_server: - `-i` option passes all remaining arguments to action - delete unused "touch file" option Views: - delete broken links - add tabs to every page - separate admin tabs from user tabs --- actions/email_server | 19 ++--- plinth/modules/email_server/__init__.py | 2 +- plinth/modules/email_server/audit/__init__.py | 5 +- plinth/modules/email_server/audit/home.py | 71 +++++++++++++++++++ .../modules/email_server/templates/alias.html | 18 +---- .../email_server/templates/email_server.html | 6 +- .../email_server/templates/form_base.html | 19 +++++ .../email_server/templates/my_mail.html | 32 +++++++++ plinth/modules/email_server/urls.py | 6 +- plinth/modules/email_server/views.py | 64 +++++++++++++---- 10 files changed, 194 insertions(+), 48 deletions(-) create mode 100644 plinth/modules/email_server/audit/home.py create mode 100644 plinth/modules/email_server/templates/form_base.html create mode 100644 plinth/modules/email_server/templates/my_mail.html diff --git a/actions/email_server b/actions/email_server index 39b39610c..1cc122e5d 100755 --- a/actions/email_server +++ b/actions/email_server @@ -32,9 +32,9 @@ def main(): parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-i', nargs=2, dest='ipc') - group.add_argument('-t', nargs=1, dest='touch_file') + group.add_argument('-i', nargs='+', dest='ipc') + # Select the first non-empty dict item adict = vars(parser.parse_args()) generator = (kv for kv in adict.items() if kv[1] is not None) subcommand, arguments = next(generator) @@ -49,7 +49,7 @@ def main(): @reserved_for_root -def subcommand_ipc(module_name, action_name): +def subcommand_ipc(module_name, action_name, *args): import plinth.modules.email_server.audit as audit # We only run actions defined in the audit module @@ -63,18 +63,7 @@ def subcommand_ipc(module_name, action_name): logger.critical('Bad action: %s/%r', module_name, action_name) sys.exit(EXIT_SYNTAX) - function() - - -def subcommand_touch_file(path): - import pathlib - - if os.getuid() == 0: - logger.critical('Do not run the `-t` option as root') - sys.exit(EXIT_PERM) - - # mode is influenced by umask - pathlib.Path(path).touch(mode=0o660, exist_ok=True) + function(*args) def _log_additional_info(): diff --git a/plinth/modules/email_server/__init__.py b/plinth/modules/email_server/__init__.py index 15fe9df30..4a8137cae 100644 --- a/plinth/modules/email_server/__init__.py +++ b/plinth/modules/email_server/__init__.py @@ -63,7 +63,7 @@ class EmailServerApp(plinth.app.App): name=info.name, short_description=info.short_description, icon='roundcube', - url=reverse_lazy('email_server:my_aliases'), + url=reverse_lazy('email_server:my_mail'), clients=manifest.clients, login_required=True ) diff --git a/plinth/modules/email_server/audit/__init__.py b/plinth/modules/email_server/audit/__init__.py index 41f1b99c6..4adc7489b 100644 --- a/plinth/modules/email_server/audit/__init__.py +++ b/plinth/modules/email_server/audit/__init__.py @@ -3,8 +3,9 @@ Provides diagnosis and repair of email server configuration issues """ -from . import ldap from . import domain +from . import home +from . import ldap from . import spam -__all__ = ['ldap', 'domain', 'spam'] +__all__ = ['domain', 'home', 'ldap', 'spam'] diff --git a/plinth/modules/email_server/audit/home.py b/plinth/modules/email_server/audit/home.py new file mode 100644 index 000000000..dc14a61c4 --- /dev/null +++ b/plinth/modules/email_server/audit/home.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +import logging +import os +import pwd +import subprocess + +from django.core.exceptions import ValidationError +from django.utils.translation import ugettext_lazy as _ +from plinth.actions import superuser_run +from plinth.errors import ActionError + +logger = logging.getLogger(__name__) + + +def exists_nam(username): + """Returns True if the user's home directory exists""" + try: + passwd = pwd.getpwnam(username) + except KeyError as e: + raise ValidationError(_('User does not exist')) from e + return _exists(passwd) + + +def exists_uid(uid_number): + """Returns True if the user's home directory exists""" + try: + passwd = pwd.getpwuid(uid_number) + except KeyError as e: + raise ValidationError(_('User does not exist')) from e + return _exists(passwd) + + +def _exists(passwd): + return os.path.exists(passwd.pw_dir) + + +def put_nam(username): + """Create a home directory for the user (identified by username)""" + _put('nam', username) + + +def put_uid(uid_number): + """Create a home directory for the user (identified by UID)""" + _put('uid', str(uid_number)) + + +def _put(arg_type, user_info): + try: + args = ['-i', 'home', 'mk', arg_type, user_info] + superuser_run('email_server', args) + except ActionError as e: + raise RuntimeError('Action script failure') from e + + +def action_mk(arg_type, user_info): + if arg_type == 'nam': + passwd = pwd.getpwnam(user_info) + elif arg_type == 'uid': + passwd = pwd.getpwuid(int(user_info)) + else: + raise ValueError('Unknown arg_type') + + args = ['sudo', '-n', '--user=#' + str(passwd.pw_uid)] + args.extend(['/bin/sh', '-c', 'mkdir -p ~']) + completed = subprocess.run(args, capture_output=True) + if completed.returncode != 0: + logger.critical('Subprocess returned %d', completed.returncode) + logger.critical('Stdout: %r', completed.stdout) + logger.critical('Stderr: %r', completed.stderr) + raise OSError('Could not create home directory') diff --git a/plinth/modules/email_server/templates/alias.html b/plinth/modules/email_server/templates/alias.html index cab40ab5c..1980b66f6 100644 --- a/plinth/modules/email_server/templates/alias.html +++ b/plinth/modules/email_server/templates/alias.html @@ -1,24 +1,12 @@ {# SPDX-License-Identifier: AGPL-3.0-or-later #} -{% extends "app.html" %} +{% extends "form_base.html" %} {% load bootstrap %} {% load i18n %} -{% block configuration %} +{% block content %} - {{ tabs|safe }} -

{% trans "Alias Management" %}

- - {% if error %} - - {% endif %} + {{ block.super }} {% if no_alias %}

{% trans "You have no email aliases." %}

diff --git a/plinth/modules/email_server/templates/email_server.html b/plinth/modules/email_server/templates/email_server.html index 89557bdb5..9f0a3125d 100644 --- a/plinth/modules/email_server/templates/email_server.html +++ b/plinth/modules/email_server/templates/email_server.html @@ -3,8 +3,12 @@ {% load i18n %} -{% block configuration %} +{% block content %} {{ tabs|safe }} + {{ block.super }} +{% endblock %} + +{% block extra_content %}

{% trans "Visit Rspamd administration interface" %} diff --git a/plinth/modules/email_server/templates/form_base.html b/plinth/modules/email_server/templates/form_base.html new file mode 100644 index 000000000..390c0936b --- /dev/null +++ b/plinth/modules/email_server/templates/form_base.html @@ -0,0 +1,19 @@ +{# SPDX-License-Identifier: AGPL-3.0-or-later #} +{% extends "base.html" %} + +{% load i18n %} + +{% block content %} + {{ tabs|safe }} + {{ block.super }} + {% if error %} +

+ {% endif %} +{% endblock %} diff --git a/plinth/modules/email_server/templates/my_mail.html b/plinth/modules/email_server/templates/my_mail.html new file mode 100644 index 000000000..7bb928151 --- /dev/null +++ b/plinth/modules/email_server/templates/my_mail.html @@ -0,0 +1,32 @@ +{# SPDX-License-Identifier: AGPL-3.0-or-later #} +{% extends "form_base.html" %} + +{% load bootstrap %} +{% load i18n %} + +{% block content %} + + {{ block.super }} + + {% if not has_homedir %} +
+ +
+ {% endif %} + +

+ Roundcube login +

+ +{% endblock %} diff --git a/plinth/modules/email_server/urls.py b/plinth/modules/email_server/urls.py index eae2a2d7c..0a5e208d8 100644 --- a/plinth/modules/email_server/urls.py +++ b/plinth/modules/email_server/urls.py @@ -6,6 +6,10 @@ from . import views urlpatterns = [ path('apps/email_server/', views.EmailServerView.as_view(), name='index'), + path('apps/email_server/security', views.TLSView.as_view()), + + path('apps/email_server/my_mail', + non_admin_view(views.MyMailView.as_view()), name='my_mail'), path('apps/email_server/my_aliases', - non_admin_view(views.AliasView.as_view()), name='my_aliases') + non_admin_view(views.AliasView.as_view())), ] diff --git a/plinth/modules/email_server/views.py b/plinth/modules/email_server/views.py index f940a3e21..7f3ab837d 100644 --- a/plinth/modules/email_server/views.py +++ b/plinth/modules/email_server/views.py @@ -3,6 +3,7 @@ import io import itertools import pwd +import plinth.utils import plinth.views from django.core.exceptions import ValidationError @@ -10,35 +11,64 @@ from django.utils.html import escape from django.utils.translation import ugettext_lazy as _ from django.views.generic.base import TemplateView -from . import forms from . import aliases +from . import audit +from . import forms -tabs = [ +admin_tabs = [ ('', _('Home')), - ('alias', _('Alias')), - ('relay', _('Relay')), - ('security', _('Security')) + ('my_mail', _('My Mail')), + ('my_aliases', _('My Aliases')) +] + +user_tabs = [ + ('my_mail', _('Home')), + ('my_aliases', _('My Aliases')) ] class EmailServerView(plinth.views.AppView): """Server configuration page""" app_id = 'email_server' - form_class = forms.EmailServerForm template_name = 'email_server.html' - def form_valid(self, form): - # old_settings = form.initial - # new_status = form.cleaned_data - # plinth.actions.superuser_run('email_server', ['--help']) - return super().form_valid(form) - def get_context_data(self, *args, **kwargs): context = super().get_context_data(*args, **kwargs) context['tabs'] = render_tabs(self.request) return context +class MyMailView(TemplateView): + template_name = 'my_mail.html' + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context['tabs'] = render_tabs(self.request) + + nam = self.request.user.username + context['has_homedir'] = audit.home.exists_nam(nam) + + return context + + def post(self, request): + try: + return self._post(request) + except ValidationError as validation_error: + context = self.get_context_data() + context['error'] = validation_error + return self.render_to_response(context, status=400) + except RuntimeError as runtime_error: + context = self.get_context_data() + context['error'] = [str(runtime_error)] + return self.render_to_response(context, status=500) + + def _post(self, request): + if not 'btn_mkhome' in request.POST: + raise ValidationError('Bad post data') + audit.home.put_nam(request.user.username) + return self.render_to_response(self.get_context_data()) + + class AliasView(TemplateView): class Checkboxes: def __init__(self, post=None, initial=None): @@ -103,6 +133,7 @@ class AliasView(TemplateView): def get_context_data(self, *args, **kwargs): context = super().get_context_data(*args, **kwargs) context['form'] = forms.AliasCreationForm() + context['tabs'] = render_tabs(self.request) uid = pwd.getpwnam(self.request.user.username).pw_uid models = aliases.get(uid) @@ -173,9 +204,16 @@ class AliasView(TemplateView): def render_tabs(request): + if plinth.utils.is_user_admin(request): + return _render_tabs(request, admin_tabs) + else: + return _render_tabs(request, user_tabs) + + +def _render_tabs(request, tab_data): sb = io.StringIO() sb.write('