From 3a8b69fc824da13139199d0dd9caf0729433037e Mon Sep 17 00:00:00 2001 From: Michael Pimmer Date: Tue, 27 Nov 2018 23:16:01 +0000 Subject: [PATCH] Backups, remote repositories: start using sshfs - actions to mount/umount via sshfs - tests Reviewed-by: James Valleroy --- actions/backups | 95 +++++++++++++++++++++++++++++- plinth/modules/backups/__init__.py | 2 +- plinth/modules/backups/sshfs.py | 38 ++++++++++++ plinth/modules/backups/views.py | 4 +- plinth/tests/.gitignore | 1 + plinth/tests/config.py | 34 +++++++++++ plinth/tests/test_backups.py | 41 +++++++++++++ 7 files changed, 210 insertions(+), 5 deletions(-) create mode 100644 plinth/modules/backups/sshfs.py create mode 100644 plinth/tests/.gitignore create mode 100644 plinth/tests/config.py diff --git a/actions/backups b/actions/backups index 84e944fc0..36d164cc6 100755 --- a/actions/backups +++ b/actions/backups @@ -32,6 +32,10 @@ from plinth.modules.backups import MANIFESTS_FOLDER, REPOSITORY TIMEOUT = 5 +class AlreadyMountedError(Exception): + pass + + def parse_arguments(): """Return parsed command line arguments as dictionary.""" parser = argparse.ArgumentParser() @@ -48,6 +52,18 @@ def parse_arguments(): init.add_argument('--encryption', help='Enryption of the repository', required=True) + # TODO: ssh commands should be in SSH or a separate action script + mount = subparsers.add_parser('mount', help='mount an ssh filesystem') + mount.add_argument('--mountpoint', help='Local mountpoint', required=True) + umount = subparsers.add_parser('umount', + help='unmount an ssh filesystem') + umount.add_argument('--mountpoint', help='Mountpoint to unmount', + required=True) + is_mounted = subparsers.add_parser('is-mounted', + help='Check whether an sshfs is mouned') + is_mounted.add_argument('--mountpoint', help='Mountpoint to check', + required=True) + list_repo = subparsers.add_parser('list-repo', help='List repository contents') @@ -70,7 +86,7 @@ def parse_arguments(): # TODO: add parameters to missing commands (export etc) for cmd in [info, init, list_repo, create_archive, delete_archive, - get_archive_apps]: + get_archive_apps, mount]: cmd.add_argument('--path', help='Repository or Archive path', required=False) # TODO: set required to True! cmd.add_argument('--ssh-keyfile', help='Path of private ssh key', @@ -139,6 +155,20 @@ def get_env(arguments): return env +def get_sshfs_env(arguments): + """Create encryption and ssh kwargs out of given arguments""" + env = dict(os.environ, BORG_RELOCATED_REPO_ACCESS_IS_OK='yes') + if arguments.encryption_passphrase: + env['BORG_PASSPHRASE'] = arguments.encryption_passphrase + if arguments.ssh_keyfile: + env['SSHKEY'] = arguments.ssh_keyfile + else: + password = read_password() + if password: + env['SSHPASS'] = password + return env + + def subcommand_init(arguments): env = get_env(arguments) init(arguments.repository, arguments.encryption, env=env) @@ -150,6 +180,66 @@ def subcommand_info(arguments): run(['borg', 'info', '--json', arguments.repository], env=env) +def subcommand_mount(arguments): + """Show repository information.""" + try: + validate_mountpoint(arguments.mountpoint) + except AlreadyMountedError: + return + + env = get_env(arguments) + remote_path = arguments.path + kwargs = {} + # the shell would expand ~/ to the local home directory + remote_path = remote_path.replace('~/', '').replace('~', '') + cmd = ['sshfs', remote_path, arguments.mountpoint, '-o', + 'UserKnownHostsFile=/dev/null', '-o', + 'StrictHostKeyChecking=no'] + timeout = None + if 'SSHPASS' in env: + cmd += ['-o', 'password_stdin'] + kwargs['input'] = env['SSHPASS'].encode() + elif 'SSHKEY' in env: + cmd += ['-o', 'IdentityFile=$SSHKEY'] + timeout = TIMEOUT + else: + raise ValueError('mount requires either SSHPASS or SSHKEY in env') + subprocess.run(cmd, check=True, env=env, timeout=timeout, **kwargs) + + +def subcommand_umount(arguments): + """Show repository information.""" + run(['umount', arguments.mountpoint]) + + +def validate_mountpoint(mountpoint): + """Check that the folder is empty, and create it if it doesn't exist""" + if os.path.exists(mountpoint): + if _is_mounted(mountpoint): + raise AlreadyMountedError('Mountpoint %s already mounted' % + mountpoint) + if os.listdir(mountpoint) or not os.path.isdir(mountpoint): + raise ValueError('Mountpoint %s is not an empty directory' % + mountpoint) + else: + os.makedirs(mountpoint) + + +def _is_mounted(mountpoint): + """Return boolean whether a local directory is a mountpoint.""" + cmd = ['mountpoint', '-q', mountpoint] + # mountpoint exits with status non-zero if it didn't find a mountpoint + try: + subprocess.run(cmd, check=True) + return True + except subprocess.CalledProcessError: + return False + + +def subcommand_is_mounted(arguments): + print(json.dumps(_is_mounted(arguments.mountpoint))) + + def subcommand_list_repo(arguments): """List repository contents.""" env = get_env(arguments) @@ -207,7 +297,6 @@ def _read_archive_file(archive, filepath): def subcommand_get_archive_apps(arguments): """Get list of apps included in archive.""" - import ipdb; ipdb.set_trace() env = get_env(arguments) manifest_folder = os.path.relpath(MANIFESTS_FOLDER, '/') borg_call = ['borg', 'list', arguments.path, manifest_folder, @@ -301,7 +390,7 @@ def run(cmd, env=None): # If the remote server asks for a password but no password is # provided, we get stuck at asking the password. timeout = None - if 'BORG_RSH' in env and 'SSHPASS' not in env: + if env and 'BORG_RSH' in env and 'SSHPASS' not in env: timeout = TIMEOUT subprocess.run(cmd, check=True, env=env, timeout=timeout) diff --git a/plinth/modules/backups/__init__.py b/plinth/modules/backups/__init__.py index 495ebbaf8..0a5fa3ba0 100644 --- a/plinth/modules/backups/__init__.py +++ b/plinth/modules/backups/__init__.py @@ -33,7 +33,7 @@ from . import api version = 1 -managed_packages = ['borgbackup'] +managed_packages = ['borgbackup', 'sshfs'] name = _('Backups') diff --git a/plinth/modules/backups/sshfs.py b/plinth/modules/backups/sshfs.py new file mode 100644 index 000000000..d3aa32b24 --- /dev/null +++ b/plinth/modules/backups/sshfs.py @@ -0,0 +1,38 @@ +# +# This file is part of FreedomBox. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +""" +Utilities to work with sshfs. +""" + +import json + +from plinth.modules.backups import run + + +def mount(remote_path, mountpoint, access_params): + run(['mount', '--mountpoint', mountpoint, '--path', remote_path], + access_params) + + +def umount(mountpoint): + run(['umount', '--mountpoint', mountpoint]) + + +def is_mounted(mountpoint): + output = run(['is-mounted', '--mountpoint', mountpoint]) + return json.loads(output) diff --git a/plinth/modules/backups/views.py b/plinth/modules/backups/views.py index c81605032..f2d203efe 100644 --- a/plinth/modules/backups/views.py +++ b/plinth/modules/backups/views.py @@ -75,7 +75,7 @@ class IndexView(TemplateView): context['title'] = backups.name context['description'] = backups.description context['info'] = backups.get_info(REPOSITORY) - context['archives'] = backups.list_archives() + context['archives'] = backups.list_archives(REPOSITORY) context['subsubmenu'] = subsubmenu apps = api.get_all_apps_for_backup() context['available_apps'] = [app.name for app in apps] @@ -300,6 +300,8 @@ class RepositoriesView(TemplateView): """Return additional context for rendering the template.""" context = super().get_context_data(**kwargs) context['title'] = 'Backup repositories' + # TODO: rename backups_repositories to something more generic, + # that can be used/managed by the storage module too repositories = kvstore.get_default('backups_repositories', []) context['repositories'] = json.loads(repositories) context['subsubmenu'] = subsubmenu diff --git a/plinth/tests/.gitignore b/plinth/tests/.gitignore new file mode 100644 index 000000000..50313bbc8 --- /dev/null +++ b/plinth/tests/.gitignore @@ -0,0 +1 @@ +config_local.py diff --git a/plinth/tests/config.py b/plinth/tests/config.py new file mode 100644 index 000000000..14c2f04a3 --- /dev/null +++ b/plinth/tests/config.py @@ -0,0 +1,34 @@ +# +# This file is part of FreedomBox. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +""" +Configuration for running tests. + +To customize these settings, create a 'config_local.py' and override +the variables defined here. +""" + +backups_ssh_path = None +backups_ssh_password = None +backups_ssh_keyfile = None +backups_ssh_mountpoint = '/mnt/plinth_test_sshfs' + +# Import config_local to override the default variables +try: + from config_local.py import * +except ImportError: + pass diff --git a/plinth/tests/test_backups.py b/plinth/tests/test_backups.py index 4da17b21f..833fa9b7e 100644 --- a/plinth/tests/test_backups.py +++ b/plinth/tests/test_backups.py @@ -25,8 +25,16 @@ import unittest from plinth import cfg from plinth.modules import backups +from plinth.modules.backups import sshfs from plinth import actions +from . import config + +try: + from . import config_local as config +except ImportError: + from . import config + euid = os.geteuid() @@ -108,3 +116,36 @@ class TestBackups(unittest.TestCase): backups.delete_archive(archive_path) content = backups.list_archives(repo_path) assert len(content) == 0 + + @unittest.skipUnless(euid == 0, 'Needs to be root') + def test_is_mounted(self): + assert not sshfs.is_mounted(self.action_directory.name) + assert sshfs.is_mounted('/') + + @unittest.skipUnless(euid == 0, 'Needs to be root') + def test_mount(self): + """Test (un)mounting if credentials for a remote location are given""" + import ipdb; ipdb.set_trace() + if config.backups_ssh_path: + access_params = self.get_remote_access_params() + if not access_params: + return + mountpoint = config.backups_ssh_mountpoint + ssh_path = config.backups_ssh_path + + sshfs.mount(ssh_path, mountpoint, access_params) + assert sshfs.is_mounted(mountpoint) + sshfs.umount(mountpoint) + assert not sshfs.is_mounted(mountpoint) + + def get_remote_access_params(self): + """ + Get access params for a remote location. + Return an empty dict if no valid access params are found. + """ + access_params = {} + if config.backups_ssh_password: + access_params['ssh_password'] = config.backups_ssh_password + elif config.backups_ssh_keyfile: + access_params['ssh_keyfile'] = config.backups_ssh_keyfile + return access_params