diff --git a/plinth/modules/ssh/__init__.py b/plinth/modules/ssh/__init__.py
index e5e40a6dc..88c18df5e 100644
--- a/plinth/modules/ssh/__init__.py
+++ b/plinth/modules/ssh/__init__.py
@@ -12,6 +12,7 @@ from plinth import menu
 from plinth.daemon import Daemon
 from plinth.modules.backups.components import BackupRestore
 from plinth.modules.firewall.components import Firewall
+from plinth.modules.users.components import UsersAndGroups
 from plinth.package import Packages
 
 from . import manifest, privileged
@@ -29,7 +30,7 @@ class SSHApp(app_module.App):
 
     app_id = 'ssh'
 
-    _version = 1
+    _version = 2
 
     def __init__(self):
         """Create components for the app."""
@@ -56,6 +57,13 @@ class SSHApp(app_module.App):
         daemon = Daemon('daemon-ssh', 'ssh')
         self.add(daemon)
 
+        groups = {
+            'freedombox-ssh': _('Remotely login using Secure Shell (SSH)')
+        }
+        users_and_groups = UsersAndGroups('users-and-groups-ssh',
+                                          groups=groups)
+        self.add(users_and_groups)
+
         backup_restore = BackupRestore('backup-restore-ssh', **manifest.backup)
         self.add(backup_restore)
 
@@ -63,7 +71,10 @@ class SSHApp(app_module.App):
         """Install and configure the app."""
         super().setup(old_version)
         privileged.setup()
-        self.enable()
+        if not old_version:
+            self.enable()
+        elif old_version == 1:
+            privileged.restrict_users(True)
 
 
 def get_host_keys():
diff --git a/plinth/modules/ssh/manifest.py b/plinth/modules/ssh/manifest.py
index 9ab3b80fa..1a772cf31 100644
--- a/plinth/modules/ssh/manifest.py
+++ b/plinth/modules/ssh/manifest.py
@@ -4,6 +4,9 @@ Application manifest for ssh.
 """
 
 backup = {
+    'config': {
+        'files': ['/etc/ssh/sshd_config.d/freedombox.conf']
+    },
     'secrets': {
         'files': [
             '/etc/ssh/ssh_host_ecdsa_key', '/etc/ssh/ssh_host_ecdsa_key.pub',
diff --git a/plinth/modules/ssh/privileged.py b/plinth/modules/ssh/privileged.py
index ee95983b0..5d7cf1ed1 100644
--- a/plinth/modules/ssh/privileged.py
+++ b/plinth/modules/ssh/privileged.py
@@ -3,6 +3,7 @@
 
 import grp
 import os
+import pathlib
 import pwd
 import shutil
 import stat
@@ -13,6 +14,8 @@ import augeas
 from plinth import action_utils, utils
 from plinth.actions import privileged
 
+config_file = pathlib.Path('/etc/ssh/sshd_config.d/freedombox.conf')
+
 
 def _validate_user(username, password, must_be_admin=True):
     """Validate a user."""
@@ -53,6 +56,23 @@ def setup():
     action_utils.dpkg_reconfigure('openssh-server', {})
 
 
+@privileged
+def restrict_users(should_restrict: bool):
+    """Restrict SSH logins to groups root, admin and freedombox-ssh."""
+    if not should_restrict:
+        config_file.unlink(missing_ok=True)
+    else:
+        config_file.write_text('AllowGroups root admin freedombox-ssh\n',
+                               encoding='utf-8')
+
+    action_utils.service_reload('sshd')
+
+
+def are_users_restricted() -> bool:
+    """Return whether only restricted groups of users are allowed."""
+    return config_file.exists()
+
+
 def get_user_homedir(username):
     """Return the home dir of a user by looking up in password database."""
     try: