matrix-synapse: Auto configure STUN/TURN using coturn server

- Matrix Synapse will automatically pick up the locally installed coturn server
during its installation. This handles only the case where coturn is installed
and configured with a valid TLS domain name before matrix-synapse is installed.

- Allow overriding STUN/TURN config. Matrix Synapse uses the local coturn
server's configuration by default. However, an administrator can override the
STUN/TURN configuration from FreedomBox web interface. Allow administrator's
overrides to co-exist with FreedomBox's managed STUN/TURN configuration.
Administrator's configuration, if it exists, always overrides FreedomBox's
managed configuration. Any updates to FreedomBox's managed configuration will
have no impact on the administrator's configuration since the latter takes
precedence.

Sunil:

- Collapse multiple turn actions into a single one for simplicity. Sending empty
configuration means removal of the configuration.

- Ensure that when removing configuration file is idempotent.

- Manage TURN configuration even when app setup is not yet completed. This fixes
issue with TURN configuration not getting setup on app installation and setup.

- Fix issue with TURN configuration getting updated on form submission even when
the field is not changed. This is due to mismatch between the browser submitted
\r\n with the internal \n.

- Simplify JavaScript for the form and attach handlers only after DOM is ready.

- Drop the no-JS message since the loss of functionality is trivial and to
reduce translation burden.

- Fix issue with URIs and secret parameters not getting updated unless the
managed checkbox changes.

- Drop specialized success messages for TURN configuration update to reduce
translation burden.

Signed-off-by: Joseph Nuthalapati <njoseph@riseup.net>
Signed-off-by: Sunil Mohan Adapa <sunil@medhas.org>
Reviewed-by: Sunil Mohan Adapa <sunil@medhas.org>

actions/matrixsynapse

@@ -5,7 +5,10 @@ Configuration helper for Matrix-Synapse server.
 """
 
 import argparse
+import json
+import os
 import pathlib
+import sys
 
 import yaml
 
@@ -14,6 +17,9 @@ from plinth.modules.matrixsynapse import (LISTENERS_CONF_PATH, ORIG_CONF_PATH,
                                           REGISTRATION_CONF_PATH,
                                           STATIC_CONF_PATH)
 
+TURN_CONF_PATH = '/etc/matrix-synapse/conf.d/freedombox-turn.yaml'
+OVERRIDDEN_TURN_CONF_PATH = '/etc/matrix-synapse/conf.d/turn.yaml'
+
 STATIC_CONFIG = {
     'max_upload_size':
         '100M',
@@ -53,6 +59,14 @@ def parse_arguments():
         'move-old-conf',
         help='Move old configuration file to backup before reinstall')
 
+    turn = subparsers.add_parser(
+        'configure-turn',
+        help='Configure a TURN server for use with Matrix Synapse')
+    turn.add_argument(
+        '--managed', required=False, default=False, action='store_true',
+        help='Whether configuration is provided by user or auto-managed by '
+        'FreedomBox')
+
     subparsers.required = True
     return parser.parse_args()
 
@@ -112,8 +126,7 @@ def subcommand_public_registration(argument):
     with open(REGISTRATION_CONF_PATH, 'w') as reg_conf_file:
         yaml.dump(config, reg_conf_file)
 
-    if action_utils.service_is_running('matrix-synapse'):
-        action_utils.service_restart('matrix-synapse')
+    action_utils.service_try_restart('matrix-synapse')
 
 
 def subcommand_move_old_conf(_arguments):
@@ -124,6 +137,39 @@ def subcommand_move_old_conf(_arguments):
         conf_file.replace(backup_file)
 
 
+def _set_turn_config(conf_file):
+    turn_server_config = json.loads(''.join(sys.stdin))
+
+    if not turn_server_config['uris']:
+        # No valid configuration, remove the configuration file
+        try:
+            os.remove(conf_file)
+        except FileNotFoundError:
+            pass
+
+        return
+
+    config = {
+        'turn_uris': turn_server_config['uris'],
+        'turn_shared_secret': turn_server_config['shared_secret'],
+        'turn_user_lifetime': 86400000,
+        'turn_allow_guests': True
+    }
+
+    with open(conf_file, 'w+') as turn_config:
+        yaml.dump(config, turn_config)
+
+
+def subcommand_configure_turn(arguments):
+    """Set parameters for the STUN/TURN server to use with Matrix Synapse."""
+    if arguments.managed:
+        _set_turn_config(TURN_CONF_PATH)
+    else:
+        _set_turn_config(OVERRIDDEN_TURN_CONF_PATH)
+
+    action_utils.service_try_restart('matrix-synapse')
+
+
 def main():
     arguments = parse_arguments()
     sub_command = arguments.subcommand.replace('-', '_')

plinth/modules/matrixsynapse/__init__.py

@@ -6,6 +6,7 @@ FreedomBox app to configure matrix-synapse server.
 import logging
 import os
 import pathlib
+from typing import List
 
 from django.urls import reverse_lazy
 from django.utils.translation import ugettext_lazy as _
@@ -17,12 +18,14 @@ from plinth import frontpage, menu
 from plinth.daemon import Daemon
 from plinth.modules.apache.components import Webserver
 from plinth.modules.backups.components import BackupRestore
+from plinth.modules.coturn.components import TurnConfiguration, TurnConsumer
 from plinth.modules.firewall.components import Firewall
 from plinth.modules.letsencrypt.components import LetsEncrypt
+from plinth.utils import is_non_empty_file
 
 from . import manifest
 
-version = 6
+version = 7
 
 managed_services = ['matrix-synapse']
 
@@ -44,14 +47,19 @@ _description = [
       '<a href="https://element.io/">Element</a> client is recommended.')
 ]
 
+depends = ['coturn']
+
 logger = logging.getLogger(__name__)
 
-SERVER_NAME_PATH = "/etc/matrix-synapse/conf.d/server_name.yaml"
+CONF_DIR = "/etc/matrix-synapse/conf.d/"
+
 ORIG_CONF_PATH = '/etc/matrix-synapse/homeserver.yaml'
-STATIC_CONF_PATH = '/etc/matrix-synapse/conf.d/freedombox-static.yaml'
-LISTENERS_CONF_PATH = '/etc/matrix-synapse/conf.d/freedombox-listeners.yaml'
-REGISTRATION_CONF_PATH = \
-    '/etc/matrix-synapse/conf.d/freedombox-registration.yaml'
+SERVER_NAME_PATH = CONF_DIR + 'server_name.yaml'
+STATIC_CONF_PATH = CONF_DIR + 'freedombox-static.yaml'
+LISTENERS_CONF_PATH = CONF_DIR + 'freedombox-listeners.yaml'
+REGISTRATION_CONF_PATH = CONF_DIR + 'freedombox-registration.yaml'
+TURN_CONF_PATH = CONF_DIR + 'freedombox-turn.yaml'
+OVERRIDDEN_TURN_CONF_PATH = CONF_DIR + 'turn.yaml'
 
 app = None
 
@@ -110,6 +118,17 @@ class MatrixSynapseApp(app_module.App):
                                        **manifest.backup)
         self.add(backup_restore)
 
+        turn = MatrixSynapseTurnConsumer('turn-matrixsynapse')
+        self.add(turn)
+
+
+class MatrixSynapseTurnConsumer(TurnConsumer):
+    """Component to manage Coturn configuration for Matrix Synapse."""
+
+    def on_config_change(self, config: TurnConfiguration):
+        """Add or update STUN/TURN configuration."""
+        update_turn_configuration(config)
+
 
 def setup(helper, old_version=None):
     """Install and configure the module."""
@@ -125,6 +144,10 @@ def setup(helper, old_version=None):
 
     app.get_component('letsencrypt-matrixsynapse').setup_certificates()
 
+    # Configure STUN/TURN only if there's a valid TLS domain set for Coturn
+    config = app.get_component('turn-matrixsynapse').get_configuration()
+    update_turn_configuration(config, force=True)
+
 
 def upgrade(helper):
     """Upgrade matrix-synapse configuration to avoid conffile prompt."""
@@ -171,7 +194,21 @@ def get_configured_domain_name():
     return config['server_name']
 
 
-def get_public_registration_status():
+def get_turn_configuration() -> (List[str], str, bool):
+    """Return TurnConfiguration if setup else empty."""
+    for file_path, managed in ((OVERRIDDEN_TURN_CONF_PATH, False),
+                               (TURN_CONF_PATH, True)):
+        if is_non_empty_file(file_path):
+            with open(file_path) as config_file:
+                config, _, _ = load_yaml_guess_indent(config_file)
+                return (TurnConfiguration(None, config['turn_uris'],
+                                          config['turn_shared_secret']),
+                        managed)
+
+    return (TurnConfiguration(), True)
+
+
+def get_public_registration_status() -> bool:
     """Return whether public registration is enabled."""
     output = actions.superuser_run('matrixsynapse',
                                    ['public-registration', 'status'])
@@ -185,3 +222,16 @@ def get_certificate_status():
         return 'no-domains'
 
     return list(status.values())[0]
+
+
+def update_turn_configuration(config: TurnConfiguration, managed=True,
+                              force=False):
+    """Update the STUN/TURN server configuration."""
+    setup_helper = globals()['setup_helper']
+    if not force and setup_helper.get_state() == 'needs-setup':
+        return
+
+    params = ['configure-turn']
+    params += ['--managed'] if managed else []
+    actions.superuser_run('matrixsynapse', params,
+                          input=config.to_json().encode())

plinth/modules/matrixsynapse/forms.py

@@ -4,8 +4,11 @@ Forms for the Matrix Synapse module.
 """
 
 from django import forms
+from django.urls import reverse_lazy
 from django.utils.translation import ugettext_lazy as _
 
+from plinth.utils import format_lazy
+
 
 class MatrixSynapseForm(forms.Form):
     enable_public_registration = forms.BooleanField(
@@ -13,3 +16,28 @@ class MatrixSynapseForm(forms.Form):
             'Enabling public registration means that anyone on the Internet '
             'can register a new account on your Matrix server. Disable this '
             'if you only want existing users to be able to use it.'))
+
+    enable_managed_turn = forms.BooleanField(
+        label=_('Automatically manage audio/video call setup'), required=False,
+        help_text=format_lazy(
+            _('Configures the local <a href={coturn_url}>coturn</a> app as '
+              'the STUN/TURN server for Matrix Synapse. Disable this if you '
+              'want to use a different STUN/TURN server.'),
+            coturn_url=reverse_lazy('coturn:index')))
+
+    # STUN/TURN server setup
+    turn_uris = forms.CharField(
+        label=_('STUN/TURN Server URIs'), required=False, strip=True,
+        widget=forms.Textarea(attrs={'rows': 4}),
+        help_text=_('List of public URIs of the STUN/TURN server, one on each '
+                    'line.'))
+
+    shared_secret = forms.CharField(
+        label=_('Shared Authentication Secret'), required=False, strip=True,
+        help_text=_('Shared secret used to compute passwords for the '
+                    'TURN server.'))
+
+    def clean_turn_uris(self):
+        """Normalize newlines in URIs."""
+        data = self.cleaned_data['turn_uris']
+        return '\n'.join([uri.strip() for uri in data.splitlines()])

plinth/modules/matrixsynapse/static/matrixsynapse.js

@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * @licstart The following is the entire license notice for the JavaScript
+ * code in this page.
+ *
+ * This file is part of FreedomBox.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @licend The above is the entire license notice for the JavaScript code
+ * in this page.
+ */
+
+jQuery(function($) {
+    $('#id_enable_managed_turn').change(function() {
+        if($(this).prop('checked')) {
+            $('#id_turn_uris').closest('.form-group').hide();
+            $('#id_shared_secret').closest('.form-group').hide();
+        } else {
+            $('#id_turn_uris').closest('.form-group').show();
+            $('#id_shared_secret').closest('.form-group').show();
+        }
+    }).change();
+});

plinth/modules/matrixsynapse/templates/matrix-synapse.html

@@ -4,6 +4,7 @@
 {% endcomment %}
 
 {% load i18n %}
+{% load static %}
 
 {% block description %}
   {% for paragraph in description %}
@@ -36,3 +37,8 @@
     </div>
   {% endif %}
 {% endblock %}
+
+{% block page_js %}
+  <script type="text/javascript"
+          src="{% static 'matrixsynapse/matrixsynapse.js' %}"></script>
+{% endblock %}

plinth/modules/matrixsynapse/tests/conftest.py

@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+Common test fixtures for Matrix Synapse.
+"""
+
+import importlib
+import pathlib
+import types
+from unittest.mock import patch
+
+import pytest
+
+current_directory = pathlib.Path(__file__).parent
+
+
+def _load_actions_module():
+    actions_file_path = str(current_directory / '..' / '..' / '..' / '..' /
+                            'actions' / 'matrixsynapse')
+    loader = importlib.machinery.SourceFileLoader('matrixsynapse',
+                                                  actions_file_path)
+    module = types.ModuleType(loader.name)
+    loader.exec_module(module)
+    return module
+
+
+actions = _load_actions_module()
+
+
+@pytest.fixture(name='managed_turn_conf_file')
+def fixture_managed_turn_conf_file(tmp_path):
+    """Returns a dummy TURN configuration file."""
+    conf_file = tmp_path / 'freedombox-turn.yaml'
+    return str(conf_file)
+
+
+@pytest.fixture(name='overridden_turn_conf_file')
+def fixture_overridden_turn_conf_file(tmp_path):
+    """Returns a dummy TURN configuration file."""
+    conf_file = tmp_path / 'turn.yaml'
+    return str(conf_file)
+
+
+@pytest.fixture(name='call_action')
+def fixture_call_action(capsys, managed_turn_conf_file,
+                        overridden_turn_conf_file):
+    """Run actions with custom root path."""
+
+    def _call_action(module_name, args, **kwargs):
+        actions.TURN_CONF_PATH = managed_turn_conf_file
+        actions.OVERRIDDEN_TURN_CONF_PATH = overridden_turn_conf_file
+        with patch('argparse._sys.argv', [module_name] +
+                   args), patch('plinth.action_utils.service_try_restart'):
+            actions.main()
+            captured = capsys.readouterr()
+            return captured.out
+
+    return _call_action

plinth/modules/matrixsynapse/tests/test_turn_config.py

@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+Test module for Matrix Synapse STUN/TURN configuration.
+"""
+
+from unittest.mock import patch
+
+import pytest
+
+from plinth.modules import matrixsynapse
+from plinth.modules.coturn.components import TurnConfiguration
+
+
+@pytest.fixture(name='test_configuration', autouse=True)
+def fixture_test_configuration(call_action, managed_turn_conf_file,
+                               overridden_turn_conf_file):
+    """Use a separate Matrix Synapse configuration for tests.
+
+    Overrides TURN configuration files and patches actions.superuser_run
+    with the fixture call_action
+    """
+    with patch('plinth.modules.matrixsynapse.TURN_CONF_PATH',
+               managed_turn_conf_file), \
+        patch('plinth.modules.matrixsynapse.OVERRIDDEN_TURN_CONF_PATH',
+              overridden_turn_conf_file), \
+        patch('plinth.modules.matrixsynapse.is_setup', return_value=True), \
+            patch('plinth.actions.superuser_run', call_action):
+        yield
+
+
+coturn_configuration = TurnConfiguration(
+    'freedombox.local', [],
+    'aiP02OAGyOlj6WGuCyqj7iaOsbkC7BUeKvKzhAsTZ8MEwMd3yTwpr2uvbOxgWe51')
+
+overridden_configuration = TurnConfiguration(
+    'public.coturn.site', [],
+    'BUeKvKzhAsTZ8MEwMd3yTwpr2uvbOxgWe51aiP02OAGyOlj6WGuCyqj7iaOsbkC7')
+
+updated_coturn_configuration = TurnConfiguration(
+    'my.freedombox.rocks', [],
+    'aiP02OsbkC7BUeKvKzhAsTZ8MEwMd3yTwpr2uvbOxgWe51AGyOlj6WGuCyqj7iaO')
+
+
+def _set_managed_configuration(monkeypatch, config=coturn_configuration):
+    monkeypatch.setattr('sys.stdin', config.to_json())
+    matrixsynapse.update_turn_configuration(config)
+
+
+def _set_overridden_configuration(monkeypatch,
+                                  config=overridden_configuration):
+    monkeypatch.setattr('sys.stdin', config.to_json())
+    matrixsynapse.update_turn_configuration(config, managed=False)
+
+
+def _assert_conf(expected_configuration, expected_managed):
+    """Assert that matrix synapse TURN configuration is as expected."""
+    config, managed = matrixsynapse.get_turn_configuration()
+    assert config.uris == expected_configuration.uris
+    assert config.shared_secret == expected_configuration.shared_secret
+    assert managed == expected_managed
+
+
+def test_managed_turn_server_configuration(monkeypatch):
+    """Test setting and getting managed TURN server configuration."""
+    _set_managed_configuration(monkeypatch)
+    _assert_conf(coturn_configuration, True)
+
+
+def test_overridden_turn_server_configuration(monkeypatch):
+    """Test setting and getting overridden TURN sever configuration."""
+    _set_overridden_configuration(monkeypatch)
+    _assert_conf(overridden_configuration, False)
+
+
+def test_revert_to_managed_turn_server_configuration(monkeypatch):
+    """Test setting and getting overridden TURN sever configuration."""
+    # Had to do all 3 operations because all fixtures were function-scoped
+    _set_managed_configuration(monkeypatch)
+    _set_overridden_configuration(monkeypatch)
+    _set_overridden_configuration(monkeypatch, TurnConfiguration())
+    _assert_conf(coturn_configuration, True)
+
+
+def test_coturn_configuration_update_after_admin_override(monkeypatch):
+    """Test that overridden conf prevails even if managed conf is updated."""
+    _set_managed_configuration(monkeypatch)
+    _set_overridden_configuration(monkeypatch)
+    _set_managed_configuration(monkeypatch, updated_coturn_configuration)
+    _assert_conf(overridden_configuration, False)

plinth/modules/matrixsynapse/views.py

@@ -12,9 +12,10 @@ from django.views.generic import FormView
 from plinth import actions
 from plinth.forms import DomainSelectionForm
 from plinth.modules import matrixsynapse, names
+from plinth.modules.coturn.components import TurnConfiguration
 from plinth.views import AppView
 
-from . import get_public_registration_status
+from . import get_public_registration_status, get_turn_configuration
 from .forms import MatrixSynapseForm
 
 
@@ -64,29 +65,62 @@ class MatrixSynapseAppView(AppView):
     def get_initial(self):
         """Return the values to fill in the form."""
         initial = super().get_initial()
+        config, managed = get_turn_configuration()
         initial.update({
             'enable_public_registration': get_public_registration_status(),
+            'enable_managed_turn': managed,
+            'turn_uris': '\n'.join(config.uris),
+            'shared_secret': config.shared_secret
         })
         return initial
 
+    @staticmethod
+    def _handle_public_registrations(new_config):
+        if new_config['enable_public_registration']:
+            actions.superuser_run('matrixsynapse',
+                                  ['public-registration', 'enable'])
+        else:
+            actions.superuser_run('matrixsynapse',
+                                  ['public-registration', 'disable'])
+
+    @staticmethod
+    def _handle_turn_configuration(old_config, new_config):
+        if not new_config['enable_managed_turn']:
+            new_turn_uris = new_config['turn_uris'].splitlines()
+            new_shared_secret = new_config['shared_secret']
+
+            turn_config_changed = \
+                old_config['turn_uris'] != new_turn_uris or \
+                old_config['shared_secret'] != new_shared_secret
+
+            if turn_config_changed:
+                matrixsynapse.update_turn_configuration(
+                    TurnConfiguration(None, new_turn_uris, new_shared_secret),
+                    managed=False)
+        else:
+            # Remove overridden turn configuration
+            matrixsynapse.update_turn_configuration(TurnConfiguration(),
+                                                    managed=False)
+
     def form_valid(self, form):
         """Handle valid form submission."""
         old_config = self.get_initial()
         new_config = form.cleaned_data
-        pubreg_same = old_config['enable_public_registration'] == \
-            new_config['enable_public_registration']
 
-        if not pubreg_same:
-            # note action public-registration restarts, if running now
-            if new_config['enable_public_registration']:
-                actions.superuser_run('matrixsynapse',
-                                      ['public-registration', 'enable'])
-                messages.success(self.request,
-                                 _('Public registration enabled'))
-            else:
-                actions.superuser_run('matrixsynapse',
-                                      ['public-registration', 'disable'])
-                messages.success(self.request,
-                                 _('Public registration disabled'))
+        def changed(prop):
+            return old_config[prop] != new_config[prop]
+
+        is_changed = False
+        if changed('enable_public_registration'):
+            self._handle_public_registrations(new_config)
+            is_changed = True
+
+        if changed('enable_managed_turn') or changed('turn_uris') or \
+           changed('shared_secret'):
+            self._handle_turn_configuration(old_config, new_config)
+            is_changed = True
+
+        if is_changed:
+            messages.success(self.request, _('Configuration updated'))
 
         return super().form_valid(form)