Backups, remote repositories: implement init, info and some test

- added functionality to use remote repositories
- added some tests

Reviewed-by: James Valleroy <jvalleroy@mailbox.org>

actions/backups

@@ -17,7 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 """
-Configuration helper for backups.
+Wrapper to handle backups using borg-backups.
 """
 
 import argparse
@@ -41,6 +41,10 @@ def parse_arguments():
         'setup', help='Create repository if it does not already exist')
     info = subparsers.add_parser('info', help='Show repository information')
     info.add_argument('--repository', help='Repository path', required=True)
+    init = subparsers.add_parser('init', help='Initialize a repository')
+    init.add_argument('--repository', help='Repository path', required=True)
+    init.add_argument('--encryption', help='Enryption of the repository',
+                      required=True)
     subparsers.add_parser('list', help='List repository contents')
 
     create = subparsers.add_parser('create', help='Create archive')
@@ -51,10 +55,17 @@ def parse_arguments():
     delete = subparsers.add_parser('delete', help='Delete archive')
     delete.add_argument('--name', help='Archive name', required=True)
 
-    export_help='Export archive contents as tar on stdout'
+    export_help = 'Export archive contents as tar on stdout'
     export_tar = subparsers.add_parser('export-tar', help=export_help)
     export_tar.add_argument('--name', help='Archive name)',
-                               required=True)
+                            required=True)
+    # TODO: add parameters to missing commands (list, create, delete, export)
+    for cmd in [info, init]:
+        cmd.add_argument('--ssh-keyfile', help='Path of private ssh key',
+                         default=None)
+        cmd.add_argument('--encryption-passphrase',
+                         help='Encryption passphrase',
+                         default=None)
 
     get_exported_archive_apps = subparsers.add_parser(
         'get-exported-archive-apps',
@@ -71,14 +82,14 @@ def parse_arguments():
     restore_exported_archive = subparsers.add_parser(
         'restore-exported-archive',
         help='Restore files from an exported archive')
-    restore_exported_archive.add_argument('--path',
+    restore_exported_archive.add_argument('--path', help='Tarball file path',
-            help='Tarball file path', required=True)
+                                          required=True)
 
     restore_archive = subparsers.add_parser(
         'restore-archive', help='Restore files from an archive')
     restore_archive.add_argument('--path', help='Archive path', required=True)
     restore_archive.add_argument('--destination', help='Destination',
-            required=True)
+                                 required=True)
 
     subparsers.required = True
     return parser.parse_args()
@@ -93,34 +104,65 @@ def subcommand_setup(_):
         if not os.path.exists(path):
             os.makedirs(path)
 
-        subprocess.run(['borg', 'init', '--encryption', 'none', REPOSITORY])
+        run(['borg', 'init', '--encryption', 'none', REPOSITORY])
+
+
+def init(repository, encryption, env=None):
+    """Initialize a local or remote borg repository"""
+    # TODO: verify that the repository does not exist?
+    # TODO: does remote borg also create folders if they don't exist?
+    if encryption != 'none' and 'BORG_PASSPHRASE' not in env:
+        raise ValueError('No encryption passphrase provided')
+    cmd = ['borg', 'init', '--encryption', encryption, repository]
+    run(cmd, env=env)
+
+
+def get_env(arguments):
+    """Create encryption and ssh kwargs out of given arguments"""
+    env = dict(os.environ, BORG_RELOCATED_REPO_ACCESS_IS_OK='yes')
+    if arguments.encryption_passphrase:
+        env['BORG_PASSPHRASE'] = arguments.encryption_passphrase
+    if arguments.ssh_keyfile:
+        env['BORG_RSH'] = "ssh -i %s" % arguments.ssh_keyfile
+    else:
+        password = read_password()
+        if password:
+            env['SSHPASS'] = password
+            env['BORG_RSH'] = 'sshpass -e ssh -o StrictHostKeyChecking=no'
+    return env
+
+
+def subcommand_init(arguments):
+    env = get_env(arguments)
+    init(arguments.repository, arguments.encryption, env=env)
 
 
 def subcommand_info(arguments):
     """Show repository information."""
-    run(['borg', 'info', '--json', arguments.repository])
+    env = get_env(arguments)
+    cmd = ['borg', 'info', '--json', arguments.repository]
+    run(cmd, env=env)
 
 
 def subcommand_list(_):
     """List repository contents."""
-    subprocess.run(['borg', 'list', '--json', REPOSITORY], check=True)
+    run(['borg', 'list', '--json', REPOSITORY])
 
 
 def subcommand_create(arguments):
     """Create archive."""
     paths = filter(os.path.exists, arguments.paths)
-    subprocess.run([
+    run([
         'borg',
         'create',
         '--json',
         REPOSITORY + '::' + arguments.name,
-    ] + list(paths), check=True)
+    ] + list(paths))
 
 
 def subcommand_delete(arguments):
     """Delete archive."""
-    subprocess.run(['borg', 'delete', REPOSITORY + '::' + arguments.name],
+    run(['borg', 'delete', REPOSITORY + '::' + arguments.name])
-                   check=True)
 
 
 def _extract(archive_path, destination, locations=None):
@@ -150,9 +192,7 @@ def _extract(archive_path, destination, locations=None):
 
 def subcommand_export_tar(arguments):
     """Export archive contents as tar stream on stdout."""
-    subprocess.run([
+    run(['borg', 'export-tar', REPOSITORY + '::' + arguments.name, '-'])
-        'borg', 'export-tar', REPOSITORY + '::' + arguments.name, '-'],
-        check=True)
 
 
 def _read_archive_file(archive, filepath):
@@ -165,7 +205,7 @@ def subcommand_get_archive_apps(arguments):
     """Get list of apps included in archive."""
     manifest_folder = os.path.relpath(MANIFESTS_FOLDER, '/')
     borg_call = ['borg', 'list', arguments.path, manifest_folder,
-            '--format', '{path}{NEWLINE}']
+                 '--format', '{path}{NEWLINE}']
     try:
         manifest_path = subprocess.check_output(borg_call).decode().strip()
     except subprocess.CalledProcessError:
@@ -173,8 +213,7 @@ def subcommand_get_archive_apps(arguments):
 
     manifest = None
     if manifest_path:
-        manifest_data = _read_archive_file(arguments.path,
+        manifest_data = _read_archive_file(arguments.path, manifest_path)
-                manifest_path)
         manifest = json.loads(manifest_data)
     if manifest:
         for app in _get_apps_of_manifest(manifest):
@@ -246,17 +285,12 @@ def read_password():
         return ''.join(sys.stdin)
 
 
-def run(cmd):
+def run(cmd, env=None):
-    """Pass provided passwords on to borg"""
+    """Wrap the command with ssh password or keyfile authentication"""
-    env = dict(os.environ, BORG_RELOCATED_REPO_ACCESS_IS_OK='yes')
+    # If the remote server asks for a password but no password is
-    password = read_password()
+    # provided, we get stuck at asking the password.
     timeout = None
-    if password:
+    if 'BORG_RSH' in env and 'SSHPASS' not in env:
-        env['SSHPASS'] = password
-        env['BORG_RSH'] = 'sshpass -e ssh -o StrictHostKeyChecking=no'
-    else:
-        # When no password is given ssh might ask for a password and get stuck.
-        # Use timeout to abort early.
         timeout = TIMEOUT
     subprocess.run(cmd, check=True, env=env, timeout=timeout)
 

plinth/modules/backups/__init__.py

@@ -26,6 +26,8 @@ from django.utils.translation import ugettext_lazy as _
 
 from plinth import actions
 from plinth.menu import main_menu
+from plinth.errors import ActionError
+from .errors import BorgError, BorgRepositoryDoesNotExistError
 
 from . import api
 
@@ -45,6 +47,23 @@ MANIFESTS_FOLDER = '/var/lib/plinth/backups-manifests/'
 REPOSITORY = '/var/lib/freedombox/borgbackup'
 # session variable name that stores when a backup file should be deleted
 SESSION_PATH_VARIABLE = 'fbx-backups-upload-path'
+# known errors that come up when remotely accessing a borg repository
+# 'errors' are error strings to look for in the stacktrace.
+KNOWN_ERRORS = [{
+        "errors": ["subprocess.TimeoutExpired"],
+        "message": _("Server not reachable - try providing a password."),
+        "raise_as": BorgError,
+    },
+    {
+        "errors": ["Connection refused"],
+        "message": _("Connection refused"),
+        "raise_as": BorgError,
+    },
+    {
+        "errors": ["not a valid repository", "does not exist"],
+        "message": _("Connection works - Repository does not exist"),
+        "raise_as": BorgRepositoryDoesNotExistError,
+    }]
 
 
 def init():
@@ -59,12 +78,17 @@ def setup(helper, old_version=None):
     helper.call('post', actions.superuser_run, 'backups', ['setup'])
 
 
-def get_info(repository, password=None):
+def get_info(repository, encryption_passphrase=None, ssh_password=None,
-    args = ['backups', ['info', '--repository', repository]]
+             ssh_keyfile=None):
+    args = ['info', '--repository', repository]
     kwargs = {}
-    if password is not None:
+    if ssh_password is not None:
-        kwargs['input'] = password.encode()
+        kwargs['input'] = ssh_password.encode()
-    output = actions.superuser_run(*args, **kwargs)
+    if ssh_keyfile is not None:
+        args += ['--ssh-keyfile', ssh_keyfile]
+    if encryption_passphrase is not None:
+        args += ['--encryption-passphrase', encryption_passphrase]
+    output = actions.superuser_run('backups', args, **kwargs)
     return json.loads(output)
 
 
@@ -81,6 +105,30 @@ def get_archive(name):
     return None
 
 
+def test_connection(repository, encryption_passphrase=None, ssh_password=None,
+                    ssh_keyfile=None):
+    """
+    Test connecting to a local or remote borg repository.
+    Tries to detect (and throw) some known ssh or borg errors.
+    Returns 'borg info' information otherwise.
+    """
+    try:
+        # TODO: instead of passing encryption_passphrase, ssh_password and
+        # ssh_keyfile around all the time, try using an 'options' dict.
+        message = get_info(repository,
+                           encryption_passphrase=encryption_passphrase,
+                           ssh_password=ssh_password, ssh_keyfile=ssh_keyfile)
+        return message
+    except ActionError as err:
+        caught_error = str(err)
+        for known_error in KNOWN_ERRORS:
+            for error in known_error["errors"]:
+                if error in caught_error:
+                    raise known_error["raise_as"](known_error["message"])
+        else:
+            raise
+
+
 def _backup_handler(packet):
     """Performs backup operation on packet."""
     if not os.path.exists(MANIFESTS_FOLDER):
@@ -108,6 +156,24 @@ def create_archive(name, app_names):
     api.backup_apps(_backup_handler, app_names, name)
 
 
+def create_repository(repository, encryption, encryption_passphrase=None,
+                      ssh_keyfile=None, ssh_password=None):
+    cmd = ['init', '--repository', repository, '--encryption', encryption]
+    if ssh_keyfile:
+        cmd += ['--ssh-keyfile', ssh_keyfile]
+    if encryption_passphrase:
+        cmd += ['--encryption-passphrase', encryption_passphrase]
+
+    kwargs = {}
+    if ssh_password:
+        kwargs['input'] = ssh_password.encode()
+
+    output = actions.superuser_run('backups', cmd, **kwargs)
+    if output:
+        output = json.loads(output)
+    return output
+
+
 def delete_archive(name):
     actions.superuser_run('backups', ['delete', '--name', name])
 

plinth/modules/backups/errors.py

@@ -0,0 +1,28 @@
+#
+# This file is part of FreedomBox.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from plinth.errors import PlinthError
+
+
+class BorgError(PlinthError):
+    """Generic class for borg errors that """
+    pass
+
+
+class BorgRepositoryDoesNotExistError(BorgError):
+    """Borg access to a repository works but the repository does not exist"""
+    pass

plinth/modules/backups/forms.py

@@ -88,13 +88,12 @@ class CreateRepositoryForm(forms.Form):
     repository = forms.CharField(
         label=_('SSH Repository Path'), strip=True,
         help_text=_('Path of the new repository. Example: '
-                    '<i>user@host/path/to/repo/</i>'))
+                    '<i>user@host:~/path/to/repo/</i>'))
     ssh_password = forms.CharField(
-        label=_('SSH password'), strip=True,
+        label=_('SSH server password'), strip=True,
-        help_text=_('Password of the SSH  Server.<br />'
+        help_text=_('Password of the SSH Server.<br />'
-                    'If you have set up <a href="https://www.ssh.com/ssh/key/"'
+                    'SSH key-based authentication is not yet possible.'),
-                    'target="_blank">SSH key-based authentication</a> you can '
+        widget=forms.PasswordInput(),
-                    'omit the password.'),
         required=False)
     encryption = forms.ChoiceField(
         label=_('Encryption'),
@@ -104,33 +103,33 @@ class CreateRepositoryForm(forms.Form):
               '<b>You need this password to restore a backup!</b>')),
         choices=[('repokey', 'Key in Repository'), ('none', 'None')]
         )
-    passphrase = forms.CharField(
+    encryption_passphrase = forms.CharField(
         label=_('Passphrase'),
         help_text=_('Passphrase; Only needed when using encryption.'),
         widget=forms.PasswordInput(),
         required=False
     )
-    confirm_passphrase = forms.CharField(
+    confirm_encryption_passphrase = forms.CharField(
         label=_('Confirm Passphrase'),
         help_text=_('Repeat the passphrase.'),
         widget=forms.PasswordInput(),
         required=False
     )
-    store_passphrase = forms.BooleanField(
+    store_passwords = forms.BooleanField(
-        label=_('Store passphrase on FreedomBox'),
+        label=_('Store passwords on FreedomBox'),
         help_text=format_lazy(
-            _('Store the passphrase on your {box_name}.'
+            _('Store the passwords on your {box_name}.'
-              '<br />You need to store the passphrase if you want to run '
+              '<br />You need to store passwords if you want to run '
               'recurrent backups.'), box_name=_(cfg.box_name)),
         required=False
     )
 
     def clean(self):
         cleaned_data = super(CreateRepositoryForm, self).clean()
-        passphrase = cleaned_data.get("passphrase")
+        passphrase = cleaned_data.get("encryption_passphrase")
-        confirm_passphrase = cleaned_data.get("confirm_passphrase")
+        confirm_passphrase = cleaned_data.get("confirm_encryption_passphrase")
 
         if passphrase != confirm_passphrase:
             raise forms.ValidationError(
-                "passphrase and confirm_passphrase do not match"
+                "The entered encryption passphrases do not match"
             )

plinth/modules/backups/views.py

@@ -40,11 +40,12 @@ from django.utils.translation import ugettext_lazy
 from django.views.generic import View, FormView, TemplateView
 
 from plinth import actions, kvstore
-from plinth.errors import PlinthError, ActionError
+from plinth.errors import PlinthError
 from plinth.modules import backups, storage
 
 from . import api, forms, SESSION_PATH_VARIABLE, REPOSITORY
 from .decorators import delete_tmp_backup_file
+from .errors import BorgRepositoryDoesNotExistError
 
 logger = logging.getLogger(__name__)
 
@@ -316,22 +317,41 @@ class CreateRepositoryView(SuccessMessageMixin, FormView):
     def get_context_data(self, **kwargs):
         """Return additional context for rendering the template."""
         context = super().get_context_data(**kwargs)
-        context['title'] = _('Create new repository')
+        context['title'] = _('Create remote backup repository')
         context['subsubmenu'] = subsubmenu
         return context
 
     def form_valid(self, form):
         """Restore files from the archive on valid form submission."""
+        repository = form.cleaned_data['repository']
+        encryption = form.cleaned_data['encryption']
+        encryption_passphrase = form.cleaned_data['encryption_passphrase']
+        ssh_password = form.cleaned_data['ssh_password']
+        ssh_keyfile = form.cleaned_data['ssh_keyfile']
+
+        # TODO: create borg repository if it doesn't exist
+        import pdb; pdb.set_trace()
+        try:
+            backups.test_connection(repository, ssh_password)
+        except BorgRepositoryDoesNotExistError:
+            kwargs = {}
+            if encryption_passphrase:
+                kwargs['encryption_passphrase'] = encryption_passphrase
+            if ssh_keyfile:
+                kwargs['ssh_keyfile'] = ssh_keyfile
+            backups.create_repository(repository, encryption, **kwargs)
+
         repositories = kvstore.get_default('backups_repositories', [])
         if repositories:
             repositories = json.loads(repositories)
         new_repo = {
             'uuid': str(uuid1()),
-            'repository': form.cleaned_data['repository'],
+            'repository': repository,
-            'encryption': form.cleaned_data['encryption'],
+            'encryption': encryption,
         }
-        if form.cleaned_data['store_passphrase']:
+        if form.cleaned_data['store_passwords']:
-            new_repo['passphrase'] = form.cleaned_data['passphrase']
+            new_repo['encryption_passphrase'] = \
+                    form.cleaned_data['encryption_passphrase']
         repositories.append(new_repo)
         kvstore.set('backups_repositories', json.dumps(repositories))
         return super().form_valid(form)
@@ -342,18 +362,12 @@ class TestRepositoryView(TemplateView):
     template_name = 'backups_test_repository.html'
 
     def post(self, request):
+        # TODO: add support for borg encryption
         context = self.get_context_data()
         repository = request.POST['backups-repository']
         ssh_password = request.POST['backups-ssh_password']
-        try:
+        (error, message) = backups.test_connection(repository, ssh_password)
-            info = backups.get_info(repository, password=ssh_password)
+        context["message"] = message
-        except ActionError as err:
+        context["error"] = error
-            if "subprocess.TimeoutExpired" in str(err):
-                msg = _("Server not reachable - try providing a password.")
-                context["error"] = msg
-            else:
-                context["error"] = str(err)
-        else:
-            context["message"] = info
 
         return self.render_to_response(context)

plinth/tests/test_backups.py

@@ -0,0 +1,83 @@
+#
+# This file is part of FreedomBox.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+"""
+Test the backups action script.
+"""
+
+import os
+import shutil
+import tempfile
+import unittest
+
+from plinth import cfg
+from plinth.modules import backups
+
+euid = os.geteuid()
+
+
+class TestBackups(unittest.TestCase):
+    """Test creating, reading and deleting a repository"""
+
+    @classmethod
+    def setUpClass(cls):
+        """Initial setup for all the classes."""
+        cls.action_directory = tempfile.TemporaryDirectory()
+        cls.backup_directory = tempfile.TemporaryDirectory()
+        cfg.actions_dir = cls.action_directory.name
+        actions_dir = os.path.join(os.path.dirname(__file__), '..', '..',
+                                   'actions')
+        shutil.copy(os.path.join(actions_dir, 'backups'), cfg.actions_dir)
+
+    @classmethod
+    def tearDownClass(cls):
+        """Cleanup after all the tests are completed."""
+        cls.action_directory.cleanup()
+        cls.backup_directory.cleanup()
+
+    @unittest.skipUnless(euid == 0, 'Needs to be root')
+    def test_nonexisting_repository(self):
+        nonexisting_dir = os.path.join(self.backup_directory.name,
+                                       'does_not_exist')
+        with self.assertRaises(backups.errors.BorgRepositoryDoesNotExistError):
+            backups.test_connection(nonexisting_dir)
+
+    @unittest.skipUnless(euid == 0, 'Needs to be root')
+    def test_empty_dir(self):
+        empty_dir = os.path.join(self.backup_directory.name, 'empty_dir')
+        os.mkdir(empty_dir)
+        with self.assertRaises(backups.errors.BorgRepositoryDoesNotExistError):
+            backups.test_connection(empty_dir)
+
+    @unittest.skipUnless(euid == 0, 'Needs to be root')
+    def test_create_unencrypted_repository(self):
+        repo_path = os.path.join(self.backup_directory.name, 'borgbackup')
+        backups.create_repository(repo_path, 'none')
+        info = backups.get_info(repo_path)
+        assert 'encryption' in info
+
+    @unittest.skipUnless(euid == 0, 'Needs to be root')
+    def test_create_encrypted_repository(self):
+        repo_path = os.path.join(self.backup_directory.name,
+                                 'borgbackup_encrypted')
+        passphrase = '12345'
+        # create_repository is supposed to create the folder automatically
+        # if it does not exist
+        backups.create_repository(repo_path, 'repokey',
+                                  encryption_passphrase=passphrase)
+        assert backups.get_info(repo_path, encryption_passphrase=passphrase)
+        assert backups.test_connection(repo_path,
+                                       encryption_passphrase=passphrase)