From e3817a1a316a626cedcfc371bd32b6884e20e9dd Mon Sep 17 00:00:00 2001 From: Michael Pimmer Date: Wed, 28 Nov 2018 17:01:29 +0000 Subject: [PATCH] Backups, remote repositories: integrate to backups index page - integrate remote repositories into backups index page - remove, mount and unmount repositories via UI Reviewed-by: James Valleroy --- plinth/modules/backups/__init__.py | 44 +++-- plinth/modules/backups/forms.py | 4 +- plinth/modules/backups/remote_locations.py | 180 ++++++++++++++++++ plinth/modules/backups/templates/backups.html | 25 ++- ...ository.html => backups_add_location.html} | 2 +- .../backups/templates/backups_location.inc | 83 ++++++++ ...es.html => backups_remove_repository.html} | 37 +++- plinth/modules/backups/urls.py | 24 ++- plinth/modules/backups/views.py | 142 ++++++++------ 9 files changed, 440 insertions(+), 101 deletions(-) create mode 100644 plinth/modules/backups/remote_locations.py rename plinth/modules/backups/templates/{backups_create_repository.html => backups_add_location.html} (94%) create mode 100644 plinth/modules/backups/templates/backups_location.inc rename plinth/modules/backups/templates/{backups_repositories.html => backups_remove_repository.html} (50%) diff --git a/plinth/modules/backups/__init__.py b/plinth/modules/backups/__init__.py index 0a5fa3ba0..1fae39a16 100644 --- a/plinth/modules/backups/__init__.py +++ b/plinth/modules/backups/__init__.py @@ -50,9 +50,12 @@ SESSION_PATH_VARIABLE = 'fbx-backups-upload-path' # known errors that come up when remotely accessing a borg repository # 'errors' are error strings to look for in the stacktrace. ACCESS_PARAMS = ['ssh_keyfile', 'ssh_password', 'encryption_passphrase'] +# kvstore key for storing remote locations +REMOTE_LOCATIONS_KEY = 'remote_locations' KNOWN_ERRORS = [{ "errors": ["subprocess.TimeoutExpired"], - "message": _("Server not reachable - try providing a password."), + "message": _("Connection refused - make sure you provided correct " + "credentials and the server is running."), "raise_as": BorgError, }, { @@ -62,7 +65,7 @@ KNOWN_ERRORS = [{ }, { "errors": ["not a valid repository", "does not exist"], - "message": _("Connection works - Repository does not exist"), + "message": _("Repository not found"), "raise_as": BorgRepositoryDoesNotExistError, }] @@ -86,11 +89,16 @@ def get_info(repository, access_params=None): def list_archives(repository, access_params=None): - output = run(['list-repo', '--path', repository], access_params) - return json.loads(output)['archives'] + try: + output = run(['list-repo', '--path', repository], access_params) + except ActionError as err: + reraise_known_error(err) + else: + return json.loads(output)['archives'] def get_archive(name): + # TODO: can't we get this archive directly? for archive in list_archives(): if archive['name'] == name: return archive @@ -98,6 +106,17 @@ def get_archive(name): return None +def reraise_known_error(err): + """Look whether the caught error is known and re-raise it accordingly""" + caught_error = str(err) + for known_error in KNOWN_ERRORS: + for error in known_error["errors"]: + if error in caught_error: + raise known_error["raise_as"](known_error["message"]) + else: + raise err + + def test_connection(repository, access_params=None): """ Test connecting to a local or remote borg repository. @@ -107,13 +126,7 @@ def test_connection(repository, access_params=None): try: return get_info(repository, access_params) except ActionError as err: - caught_error = str(err) - for known_error in KNOWN_ERRORS: - for error in known_error["errors"]: - if error in caught_error: - raise known_error["raise_as"](known_error["message"]) - else: - raise + reraise_known_error(err) def _backup_handler(packet): @@ -137,6 +150,7 @@ def _backup_handler(packet): paths.append(manifest_path) actions.superuser_run( 'backups', ['create-archive', '--name', packet.label, '--paths'] + + # TODO: add ssh_keyfile paths) @@ -177,8 +191,9 @@ def _restore_exported_archive_handler(packet): """Perform restore operation on packet.""" locations = {'directories': packet.directories, 'files': packet.files} locations_data = json.dumps(locations) - actions.superuser_run('backups', ['restore-exported-archive', - '--path', packet.label], input=locations_data.encode()) + actions.superuser_run('backups', ['restore-exported-archive', '--path', + packet.label], + input=locations_data.encode()) def _restore_archive_handler(packet): @@ -186,7 +201,8 @@ def _restore_archive_handler(packet): locations = {'directories': packet.directories, 'files': packet.files} locations_data = json.dumps(locations) actions.superuser_run('backups', ['restore-archive', '--path', - packet.label, '--destination', '/'], input=locations_data.encode()) + packet.label, '--destination', '/'], + input=locations_data.encode()) def restore_from_upload(path, apps=None): diff --git a/plinth/modules/backups/forms.py b/plinth/modules/backups/forms.py index 0da605950..5506dce29 100644 --- a/plinth/modules/backups/forms.py +++ b/plinth/modules/backups/forms.py @@ -84,7 +84,7 @@ class UploadForm(forms.Form): help_text=_('Select the backup file you want to upload')) -class CreateRepositoryForm(forms.Form): +class AddRepositoryForm(forms.Form): repository = forms.CharField( label=_('SSH Repository Path'), strip=True, help_text=_('Path of the new repository. Example: ' @@ -125,7 +125,7 @@ class CreateRepositoryForm(forms.Form): ) def clean(self): - cleaned_data = super(CreateRepositoryForm, self).clean() + cleaned_data = super(AddRepositoryForm, self).clean() passphrase = cleaned_data.get("encryption_passphrase") confirm_passphrase = cleaned_data.get("confirm_encryption_passphrase") diff --git a/plinth/modules/backups/remote_locations.py b/plinth/modules/backups/remote_locations.py new file mode 100644 index 000000000..c4c26ec4a --- /dev/null +++ b/plinth/modules/backups/remote_locations.py @@ -0,0 +1,180 @@ +# +# This file is part of FreedomBox. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +""" +Manage remote storage locations +""" + +import json +import logging +import os +from uuid import uuid1 + +from django.utils.translation import ugettext as _ + +from plinth import kvstore +from plinth.errors import ActionError + +from . import sshfs, list_archives, reraise_known_error, REMOTE_LOCATIONS_KEY +from .errors import BorgError + +logger = logging.getLogger(__name__) +MOUNTPOINT = '/media/' + + +def add(path, repotype, encryption, access_params, store_passwords, added_by): + locations = get_locations() + location = { + 'uuid': str(uuid1()), + 'path': path, + 'type': repotype, + 'encryption': encryption, + 'added_by': added_by + } + if store_passwords: + if 'encryption_passphrase' in access_params: + location['encryption_passphrase'] = \ + access_params['encryption_passphrase'] + if 'ssh_password' in access_params: + location['ssh_password'] = access_params['ssh_password'] + locations.append(location) + kvstore.set(REMOTE_LOCATIONS_KEY, json.dumps(locations)) + + +def delete(uuid): + """Umount a location, remove it from kvstore and unlink the mountpoint""" + locations = get_locations() + location = get_location(uuid) + mountpoint = os.path.join(MOUNTPOINT, location['uuid']) + locations = list(filter(lambda location: location['uuid'] != uuid, + locations)) + kvstore.set(REMOTE_LOCATIONS_KEY, json.dumps(locations)) + if os.path.exists(mountpoint): + try: + sshfs.umount(mountpoint) + except ActionError: + pass + try: + os.unlink(mountpoint) + except Exception as err: + logger.error(err) + + +def get_archives(uuid=None): + """ + Get archives of one or all locations. + returns: { + uuid: { + 'path': path, + 'type': type, + 'archives': [], + 'error': error_message + } + } + """ + locations = {} + for location in get_locations(): + mountpoint = os.path.join(MOUNTPOINT, location['uuid']) + new_location = { + 'path': location['path'], + 'mounted': uuid_is_mounted(location['uuid']), + } + if new_location['mounted']: + try: + new_location['archives'] = list_archives(mountpoint) + except BorgError as err: + new_location['error'] = str(err) + except Exception as err: + logger.error(err) + new_location['error'] = _("Access failed") + locations[location['uuid']] = new_location + + return locations + + +def get_locations(location_type=None): + """Get list of all locations""" + # TODO: hold locations in memory? + locations = kvstore.get_default(REMOTE_LOCATIONS_KEY, []) + if locations: + locations = json.loads(locations) + if location_type: + locations = [location for location in locations if 'type' in location + and location['type'] == location_type] + return locations + + +def get_location(uuid): + locations = get_locations() + return list(filter(lambda location: location['uuid'] == uuid, + locations))[0] + + +def _mount_locations(uuid=None): + locations = get_locations(location_type='ssh') + for location in locations: + _mount_location(location) + + +def _mount_location(location): + # TODO: shouldn't I just store and query the access_params as they are? + # but encryption_passphrase is not an ssh access_param.. + mountpoint = os.path.join(MOUNTPOINT, location['uuid']) + is_mounted = False + if sshfs.is_mounted(mountpoint): + is_mounted = True + else: + access_params = _get_access_params(location) + # TODO: use actual feedback of sshfs.mount + try: + sshfs.mount(location['path'], mountpoint, access_params) + except Exception as err: + reraise_known_error(err) + is_mounted = True + return is_mounted + + +def _umount_location(location): + mountpoint = os.path.join(MOUNTPOINT, location['uuid']) + return sshfs.umount(mountpoint) + + +def _get_access_params(location): + keys = ['encryption_passphrase', 'ssh_keyfile', 'ssh_password'] + access_params = {key: location[key] for key in keys if key in location} + if location['type'] == 'ssh': + if 'ssh_keyfile' not in location and 'ssh_password' not in \ + location: + raise ValueError('Missing credentials') + return access_params + + +def mount_uuid(uuid): + location = get_location(uuid) + mounted = False + if location: + mounted = _mount_location(location) + return mounted + + +def umount_uuid(uuid): + location = get_location(uuid) + return _umount_location(location) + + +def uuid_is_mounted(uuid): + mountpoint = os.path.join(MOUNTPOINT, uuid) + return sshfs.is_mounted(mountpoint) diff --git a/plinth/modules/backups/templates/backups.html b/plinth/modules/backups/templates/backups.html index 7aa10d2da..6379a3584 100644 --- a/plinth/modules/backups/templates/backups.html +++ b/plinth/modules/backups/templates/backups.html @@ -28,6 +28,17 @@ .share-operations { text-align: right; } + .mount-error { + padding: 0px 10px 0px 10px; + color: orange; + } + .mount-success { + padding: 0px 10px 0px 10px; + color: black; + } + .inline-block { + display: inline-block; + } {% endblock %} @@ -48,7 +59,7 @@ id="archives-list"> - {% trans "Name" %} + {{ box_name }} storage @@ -78,4 +89,16 @@ {% endif %} + {% for uuid,location in remote_archives.items %} + {% include "backups_location.inc" %} + {% endfor %} + +
+ + + {% trans '+ Add Remote Repository' %} + + {% endblock %} diff --git a/plinth/modules/backups/templates/backups_create_repository.html b/plinth/modules/backups/templates/backups_add_location.html similarity index 94% rename from plinth/modules/backups/templates/backups_create_repository.html rename to plinth/modules/backups/templates/backups_add_location.html index 0b4be62eb..d125b1cc7 100644 --- a/plinth/modules/backups/templates/backups_create_repository.html +++ b/plinth/modules/backups/templates/backups_add_location.html @@ -34,7 +34,7 @@ value="{% trans "Submit" %}"/> + formaction="{% url 'backups:location-test' %}" /> {% endblock %} diff --git a/plinth/modules/backups/templates/backups_location.inc b/plinth/modules/backups/templates/backups_location.inc new file mode 100644 index 000000000..08153c98b --- /dev/null +++ b/plinth/modules/backups/templates/backups_location.inc @@ -0,0 +1,83 @@ +{% comment %} +# +# This file is part of FreedomBox. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +{% endcomment %} + +{% load i18n %} + + + + + + + + + + + {% for archive in location.archives %} + + + + + {% endfor %} + +
+ {{ location.path }} + {% if location.mounted %} +
+ {% csrf_token %} + +
+ {% else %} + +
+ {% csrf_token %} + +
+ {% endif %} + + + 		+
{{ archive.name }} + + {% trans "Download" %} + + + {% trans "Restore" %} + + + + +
diff --git a/plinth/modules/backups/templates/backups_repositories.html b/plinth/modules/backups/templates/backups_remove_repository.html similarity index 50% rename from plinth/modules/backups/templates/backups_repositories.html rename to plinth/modules/backups/templates/backups_remove_repository.html index b79d6d1dd..e54c9601d 100644 --- a/plinth/modules/backups/templates/backups_repositories.html +++ b/plinth/modules/backups/templates/backups_remove_repository.html @@ -18,22 +18,39 @@ # {% endcomment %} +{% load bootstrap %} {% load i18n %} {% block content %} - -

{% trans 'Backup repositories' %}

+

{{ title }}

- {% for repository in repositories %} - Repository: {{ repository.repository }}
- {% endfor %} + + {% trans "Are you sure that you want to remove the repository" %} + {{ location.path }}? + +

+ {% blocktrans %} + The remote repository will not be deleted. + This just removes the repository from the listing on the backup page, you + can add it again later on. + {% endblocktrans %}

- - {% trans 'Add Repository' %} - +

+

+ {% csrf_token %} + + + + {% trans "Abort" %} + +
+

{% endblock %} + diff --git a/plinth/modules/backups/urls.py b/plinth/modules/backups/urls.py index 9168922cd..83dab4251 100644 --- a/plinth/modules/backups/urls.py +++ b/plinth/modules/backups/urls.py @@ -20,10 +20,10 @@ URLs for the backups module. from django.conf.urls import url -from .views import IndexView, CreateArchiveView, CreateRepositoryView, \ - DeleteArchiveView, UploadArchiveView, ExportAndDownloadView, \ - RepositoriesView, RestoreArchiveView, RestoreFromUploadView, \ - TestRepositoryView +from .views import IndexView, CreateArchiveView, AddLocationView, \ + DeleteArchiveView, ExportAndDownloadView, RemoveLocationView, \ + mount_location, umount_location, UploadArchiveView, \ + RestoreArchiveView, RestoreFromUploadView, TestLocationView urlpatterns = [ url(r'^sys/backups/$', IndexView.as_view(), name='index'), @@ -37,10 +37,14 @@ urlpatterns = [ RestoreArchiveView.as_view(), name='restore-archive'), url(r'^sys/backups/restore-from-upload/$', RestoreFromUploadView.as_view(), name='restore-from-upload'), - url(r'^sys/backups/repositories/$', - RepositoriesView.as_view(), name='repositories'), - url(r'^sys/backups/repositories/create/$', - CreateRepositoryView.as_view(), name='create-repository'), - url(r'^sys/backups/repositories/test/$', - TestRepositoryView.as_view(), name='test-repository'), + url(r'^sys/backups/locations/add$', + AddLocationView.as_view(), name='location-add'), + url(r'^sys/backups/locations/test/$', + TestLocationView.as_view(), name='location-test'), + url(r'^sys/backups/locations/delete/(?P[^/]+)/$', + RemoveLocationView.as_view(), name='location-remove'), + url(r'^sys/backups/locations/mount/(?P[^/]+)/$', + mount_location, name='location-mount'), + url(r'^sys/backups/locations/umount/(?P[^/]+)/$', + umount_location, name='location-umount'), ] diff --git a/plinth/modules/backups/views.py b/plinth/modules/backups/views.py index f2d203efe..9235566cf 100644 --- a/plinth/modules/backups/views.py +++ b/plinth/modules/backups/views.py @@ -20,13 +20,11 @@ Views for the backups app. from datetime import datetime import gzip -import json from io import BytesIO import logging import mimetypes import os import tempfile -from uuid import uuid1 from urllib.parse import unquote from django.contrib import messages @@ -39,11 +37,11 @@ from django.utils.translation import ugettext as _ from django.utils.translation import ugettext_lazy from django.views.generic import View, FormView, TemplateView -from plinth import actions, kvstore -from plinth.errors import PlinthError +from plinth import actions +from plinth.errors import PlinthError, ActionError from plinth.modules import backups, storage -from . import api, forms, SESSION_PATH_VARIABLE, REPOSITORY +from . import api, forms, SESSION_PATH_VARIABLE, REPOSITORY, remote_locations from .decorators import delete_tmp_backup_file from .errors import BorgRepositoryDoesNotExistError @@ -58,9 +56,6 @@ subsubmenu = [{ }, { 'url': reverse_lazy('backups:create'), 'text': ugettext_lazy('Create') -}, { - 'url': reverse_lazy('backups:repositories'), - 'text': ugettext_lazy('Repositories') }] @@ -69,6 +64,9 @@ class IndexView(TemplateView): """View to show list of archives.""" template_name = 'backups.html' + def get_remote_archives(self): + return {} # uuid --> archive list + def get_context_data(self, **kwargs): """Return additional context for rendering the template.""" context = super().get_context_data(**kwargs) @@ -76,9 +74,8 @@ class IndexView(TemplateView): context['description'] = backups.description context['info'] = backups.get_info(REPOSITORY) context['archives'] = backups.list_archives(REPOSITORY) + context['remote_archives'] = remote_locations.get_archives() context['subsubmenu'] = subsubmenu - apps = api.get_all_apps_for_backup() - context['available_apps'] = [app.name for app in apps] return context @@ -129,7 +126,7 @@ class DeleteArchiveView(SuccessMessageMixin, TemplateView): """Delete the archive.""" backups.delete_archive(name) messages.success(request, _('Archive deleted.')) - return redirect(reverse_lazy('backups:index')) + return redirect('backups:index') def _get_file_response(path, filename): @@ -292,29 +289,13 @@ class ExportAndDownloadView(View): return response -class RepositoriesView(TemplateView): - """View list of repositories.""" - template_name = 'backups_repositories.html' - - def get_context_data(self, **kwargs): - """Return additional context for rendering the template.""" - context = super().get_context_data(**kwargs) - context['title'] = 'Backup repositories' - # TODO: rename backups_repositories to something more generic, - # that can be used/managed by the storage module too - repositories = kvstore.get_default('backups_repositories', []) - context['repositories'] = json.loads(repositories) - context['subsubmenu'] = subsubmenu - return context - - -class CreateRepositoryView(SuccessMessageMixin, FormView): - """View to create a new repository.""" - form_class = forms.CreateRepositoryForm +class AddLocationView(SuccessMessageMixin, FormView): + """View to create a new remote backup location.""" + form_class = forms.AddRepositoryForm prefix = 'backups' - template_name = 'backups_create_repository.html' - success_url = reverse_lazy('backups:repositories') - success_message = _('Created new repository.') + template_name = 'backups_add_location.html' + success_url = reverse_lazy('backups:index') + success_message = _('Added new location.') def get_context_data(self, **kwargs): """Return additional context for rendering the template.""" @@ -329,48 +310,83 @@ class CreateRepositoryView(SuccessMessageMixin, FormView): encryption = form.cleaned_data['encryption'] encryption_passphrase = form.cleaned_data['encryption_passphrase'] ssh_password = form.cleaned_data['ssh_password'] - ssh_keyfile = form.cleaned_data['ssh_keyfile'] + store_passwords = form.cleaned_data['store_passwords'] + # TODO: add ssh_keyfile + # ssh_keyfile = form.cleaned_data['ssh_keyfile'] - # TODO: create borg repository if it doesn't exist - import pdb; pdb.set_trace() + access_params = {} + if encryption_passphrase: + access_params['encryption_passphrase'] = encryption_passphrase + if ssh_password: + access_params['ssh_password'] = ssh_password + """ + if ssh_keyfile: + access_params['ssh_keyfile'] = ssh_keyfile + """ + remote_locations.add(repository, 'ssh', encryption, access_params, + store_passwords, 'backups') + # Create the borg repository if it doesn't exist try: - backups.test_connection(repository, ssh_password) + backups.test_connection(repository, access_params) except BorgRepositoryDoesNotExistError: - access_params = {} - if encryption_passphrase: - access_params['encryption_passphrase'] = encryption_passphrase - if ssh_keyfile: - access_params['ssh_keyfile'] = ssh_keyfile backups.create_repository(repository, encryption, access_params=access_params) - - repositories = kvstore.get_default('backups_repositories', []) - if repositories: - repositories = json.loads(repositories) - new_repo = { - 'uuid': str(uuid1()), - 'repository': repository, - 'encryption': encryption, - } - if form.cleaned_data['store_passwords']: - new_repo['encryption_passphrase'] = \ - form.cleaned_data['encryption_passphrase'] - repositories.append(new_repo) - kvstore.set('backups_repositories', json.dumps(repositories)) return super().form_valid(form) -class TestRepositoryView(TemplateView): +class TestLocationView(TemplateView): """View to create a new repository.""" - template_name = 'backups_test_repository.html' + template_name = 'backups_test_location.html' def post(self, request): - # TODO: add support for borg encryption + # TODO: add support for borg encryption and ssh keyfile context = self.get_context_data() repository = request.POST['backups-repository'] - ssh_password = request.POST['backups-ssh_password'] - (error, message) = backups.test_connection(repository, ssh_password) - context["message"] = message - context["error"] = error + access_params = { + 'ssh_password': request.POST['backups-ssh_password'], + } + try: + repo_info = backups.test_connection(repository, access_params) + context["message"] = repo_info + except ActionError as err: + context["error"] = str(err) return self.render_to_response(context) + + +class RemoveLocationView(SuccessMessageMixin, TemplateView): + """View to delete an archive.""" + template_name = 'backups_remove_repository.html' + + def get_context_data(self, uuid, **kwargs): + """Return additional context for rendering the template.""" + context = super().get_context_data(**kwargs) + context['title'] = _('Remove Repository') + context['location'] = remote_locations.get_location(uuid) + return context + + def post(self, request, uuid): + """Delete the archive.""" + remote_locations.delete(uuid) + messages.success(request, _('Repository removed. The remote backup ' + 'itself was not deleted.')) + return redirect('backups:index') + + +def umount_location(request, uuid): + remote_locations.umount_uuid(uuid) + if remote_locations.uuid_is_mounted(uuid): + messages.error(request, _('Unmounting failed!')) + return redirect('backups:index') + + +def mount_location(request, uuid): + try: + remote_locations.mount_uuid(uuid) + except Exception as err: + msg = "%s: %s" % (_('Mounting failed'), str(err)) + messages.error(request, msg) + else: + if not remote_locations.uuid_is_mounted(uuid): + messages.error(request, _('Mounting failed')) + return redirect('backups:index')