Backups, remote repositories: start using sshfs

- actions to mount/umount via sshfs
- tests

Reviewed-by: James Valleroy <jvalleroy@mailbox.org>

actions/backups

@@ -32,6 +32,10 @@ from plinth.modules.backups import MANIFESTS_FOLDER, REPOSITORY
 TIMEOUT = 5
 
 
+class AlreadyMountedError(Exception):
+    pass
+
+
 def parse_arguments():
     """Return parsed command line arguments as dictionary."""
     parser = argparse.ArgumentParser()
@@ -48,6 +52,18 @@ def parse_arguments():
     init.add_argument('--encryption', help='Enryption of the repository',
                       required=True)
 
+    # TODO: ssh commands should be in SSH or a separate action script
+    mount = subparsers.add_parser('mount', help='mount an ssh filesystem')
+    mount.add_argument('--mountpoint', help='Local mountpoint', required=True)
+    umount = subparsers.add_parser('umount',
+                                   help='unmount an ssh filesystem')
+    umount.add_argument('--mountpoint', help='Mountpoint to unmount',
+                        required=True)
+    is_mounted = subparsers.add_parser('is-mounted',
+                                       help='Check whether an sshfs is mouned')
+    is_mounted.add_argument('--mountpoint', help='Mountpoint to check',
+                            required=True)
+
     list_repo = subparsers.add_parser('list-repo',
                                       help='List repository contents')
 
@@ -70,7 +86,7 @@ def parse_arguments():
 
     # TODO: add parameters to missing commands (export etc)
     for cmd in [info, init, list_repo, create_archive, delete_archive,
-                get_archive_apps]:
+                get_archive_apps, mount]:
         cmd.add_argument('--path', help='Repository or Archive path',
                          required=False)  # TODO: set required to True!
         cmd.add_argument('--ssh-keyfile', help='Path of private ssh key',
@@ -139,6 +155,20 @@ def get_env(arguments):
     return env
 
 
+def get_sshfs_env(arguments):
+    """Create encryption and ssh kwargs out of given arguments"""
+    env = dict(os.environ, BORG_RELOCATED_REPO_ACCESS_IS_OK='yes')
+    if arguments.encryption_passphrase:
+        env['BORG_PASSPHRASE'] = arguments.encryption_passphrase
+    if arguments.ssh_keyfile:
+        env['SSHKEY'] = arguments.ssh_keyfile
+    else:
+        password = read_password()
+        if password:
+            env['SSHPASS'] = password
+    return env
+
+
 def subcommand_init(arguments):
     env = get_env(arguments)
     init(arguments.repository, arguments.encryption, env=env)
@@ -150,6 +180,66 @@ def subcommand_info(arguments):
     run(['borg', 'info', '--json', arguments.repository], env=env)
 
 
+def subcommand_mount(arguments):
+    """Show repository information."""
+    try:
+        validate_mountpoint(arguments.mountpoint)
+    except AlreadyMountedError:
+        return
+
+    env = get_env(arguments)
+    remote_path = arguments.path
+    kwargs = {}
+    # the shell would expand ~/ to the local home directory
+    remote_path = remote_path.replace('~/', '').replace('~', '')
+    cmd = ['sshfs', remote_path, arguments.mountpoint, '-o',
+           'UserKnownHostsFile=/dev/null', '-o',
+           'StrictHostKeyChecking=no']
+    timeout = None
+    if 'SSHPASS' in env:
+        cmd += ['-o', 'password_stdin']
+        kwargs['input'] = env['SSHPASS'].encode()
+    elif 'SSHKEY' in env:
+        cmd += ['-o', 'IdentityFile=$SSHKEY']
+        timeout = TIMEOUT
+    else:
+        raise ValueError('mount requires either SSHPASS or SSHKEY in env')
+    subprocess.run(cmd, check=True, env=env, timeout=timeout, **kwargs)
+
+
+def subcommand_umount(arguments):
+    """Show repository information."""
+    run(['umount', arguments.mountpoint])
+
+
+def validate_mountpoint(mountpoint):
+    """Check that the folder is empty, and create it if it doesn't exist"""
+    if os.path.exists(mountpoint):
+        if _is_mounted(mountpoint):
+            raise AlreadyMountedError('Mountpoint %s already mounted' %
+                                      mountpoint)
+        if os.listdir(mountpoint) or not os.path.isdir(mountpoint):
+            raise ValueError('Mountpoint %s is not an empty directory' %
+                             mountpoint)
+    else:
+        os.makedirs(mountpoint)
+
+
+def _is_mounted(mountpoint):
+    """Return boolean whether a local directory is a mountpoint."""
+    cmd = ['mountpoint', '-q', mountpoint]
+    # mountpoint exits with status non-zero if it didn't find a mountpoint
+    try:
+        subprocess.run(cmd, check=True)
+        return True
+    except subprocess.CalledProcessError:
+        return False
+
+
+def subcommand_is_mounted(arguments):
+    print(json.dumps(_is_mounted(arguments.mountpoint)))
+
+
 def subcommand_list_repo(arguments):
     """List repository contents."""
     env = get_env(arguments)
@@ -207,7 +297,6 @@ def _read_archive_file(archive, filepath):
 
 def subcommand_get_archive_apps(arguments):
     """Get list of apps included in archive."""
-    import ipdb; ipdb.set_trace()
     env = get_env(arguments)
     manifest_folder = os.path.relpath(MANIFESTS_FOLDER, '/')
     borg_call = ['borg', 'list', arguments.path, manifest_folder,
@@ -301,7 +390,7 @@ def run(cmd, env=None):
     # If the remote server asks for a password but no password is
     # provided, we get stuck at asking the password.
     timeout = None
-    if 'BORG_RSH' in env and 'SSHPASS' not in env:
+    if env and 'BORG_RSH' in env and 'SSHPASS' not in env:
         timeout = TIMEOUT
     subprocess.run(cmd, check=True, env=env, timeout=timeout)
 

plinth/modules/backups/__init__.py

@@ -33,7 +33,7 @@ from . import api
 
 version = 1
 
-managed_packages = ['borgbackup']
+managed_packages = ['borgbackup', 'sshfs']
 
 name = _('Backups')
 

plinth/modules/backups/sshfs.py

@@ -0,0 +1,38 @@
+#
+# This file is part of FreedomBox.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+Utilities to work with sshfs.
+"""
+
+import json
+
+from plinth.modules.backups import run
+
+
+def mount(remote_path, mountpoint, access_params):
+    run(['mount', '--mountpoint', mountpoint, '--path', remote_path],
+        access_params)
+
+
+def umount(mountpoint):
+    run(['umount', '--mountpoint', mountpoint])
+
+
+def is_mounted(mountpoint):
+    output = run(['is-mounted', '--mountpoint', mountpoint])
+    return json.loads(output)

plinth/modules/backups/views.py

@@ -75,7 +75,7 @@ class IndexView(TemplateView):
         context['title'] = backups.name
         context['description'] = backups.description
         context['info'] = backups.get_info(REPOSITORY)
-        context['archives'] = backups.list_archives()
+        context['archives'] = backups.list_archives(REPOSITORY)
         context['subsubmenu'] = subsubmenu
         apps = api.get_all_apps_for_backup()
         context['available_apps'] = [app.name for app in apps]
@@ -300,6 +300,8 @@ class RepositoriesView(TemplateView):
         """Return additional context for rendering the template."""
         context = super().get_context_data(**kwargs)
         context['title'] = 'Backup repositories'
+        # TODO: rename backups_repositories to something more generic,
+        # that can be used/managed by the storage module too
         repositories = kvstore.get_default('backups_repositories', [])
         context['repositories'] = json.loads(repositories)
         context['subsubmenu'] = subsubmenu

plinth/tests/.gitignore

@@ -0,0 +1 @@
+config_local.py

plinth/tests/config.py

@@ -0,0 +1,34 @@
+#
+# This file is part of FreedomBox.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+Configuration for running tests.
+
+To customize these settings, create a 'config_local.py' and override
+the variables defined here.
+"""
+
+backups_ssh_path = None
+backups_ssh_password = None
+backups_ssh_keyfile = None
+backups_ssh_mountpoint = '/mnt/plinth_test_sshfs'
+
+# Import config_local to override the default variables
+try:
+    from config_local.py import *
+except ImportError:
+    pass

plinth/tests/test_backups.py

@@ -25,8 +25,16 @@ import unittest
 
 from plinth import cfg
 from plinth.modules import backups
+from plinth.modules.backups import sshfs
 from plinth import actions
 
+from . import config
+
+try:
+    from . import config_local as config
+except ImportError:
+    from . import config
+
 
 euid = os.geteuid()
 
@@ -108,3 +116,36 @@ class TestBackups(unittest.TestCase):
         backups.delete_archive(archive_path)
         content = backups.list_archives(repo_path)
         assert len(content) == 0
+
+    @unittest.skipUnless(euid == 0, 'Needs to be root')
+    def test_is_mounted(self):
+        assert not sshfs.is_mounted(self.action_directory.name)
+        assert sshfs.is_mounted('/')
+
+    @unittest.skipUnless(euid == 0, 'Needs to be root')
+    def test_mount(self):
+        """Test (un)mounting if credentials for a remote location are given"""
+        import ipdb; ipdb.set_trace()
+        if config.backups_ssh_path:
+            access_params = self.get_remote_access_params()
+            if not access_params:
+                return
+            mountpoint = config.backups_ssh_mountpoint
+            ssh_path = config.backups_ssh_path
+
+            sshfs.mount(ssh_path, mountpoint, access_params)
+            assert sshfs.is_mounted(mountpoint)
+            sshfs.umount(mountpoint)
+            assert not sshfs.is_mounted(mountpoint)
+
+    def get_remote_access_params(self):
+        """
+        Get access params for a remote location.
+        Return an empty dict if no valid access params are found.
+        """
+        access_params = {}
+        if config.backups_ssh_password:
+            access_params['ssh_password'] = config.backups_ssh_password
+        elif config.backups_ssh_keyfile:
+            access_params['ssh_keyfile'] = config.backups_ssh_keyfile
+        return access_params