Backups, remote repositories: integrate to backups index page

- integrate remote repositories into backups index page
- remove, mount and unmount repositories via UI

Reviewed-by: James Valleroy <jvalleroy@mailbox.org>
This commit is contained in:
Michael Pimmer 2018-11-28 17:01:29 +00:00 committed by James Valleroy
parent 3a8b69fc82
commit e3817a1a31
No known key found for this signature in database
GPG Key ID: 77C0C75E7B650808
9 changed files with 440 additions and 101 deletions

View File

@ -50,9 +50,12 @@ SESSION_PATH_VARIABLE = 'fbx-backups-upload-path'
# known errors that come up when remotely accessing a borg repository
# 'errors' are error strings to look for in the stacktrace.
ACCESS_PARAMS = ['ssh_keyfile', 'ssh_password', 'encryption_passphrase']
# kvstore key for storing remote locations
REMOTE_LOCATIONS_KEY = 'remote_locations'
KNOWN_ERRORS = [{
"errors": ["subprocess.TimeoutExpired"],
"message": _("Server not reachable - try providing a password."),
"message": _("Connection refused - make sure you provided correct "
"credentials and the server is running."),
"raise_as": BorgError,
},
{
@ -62,7 +65,7 @@ KNOWN_ERRORS = [{
},
{
"errors": ["not a valid repository", "does not exist"],
"message": _("Connection works - Repository does not exist"),
"message": _("Repository not found"),
"raise_as": BorgRepositoryDoesNotExistError,
}]
@ -86,11 +89,16 @@ def get_info(repository, access_params=None):
def list_archives(repository, access_params=None):
output = run(['list-repo', '--path', repository], access_params)
return json.loads(output)['archives']
try:
output = run(['list-repo', '--path', repository], access_params)
except ActionError as err:
reraise_known_error(err)
else:
return json.loads(output)['archives']
def get_archive(name):
# TODO: can't we get this archive directly?
for archive in list_archives():
if archive['name'] == name:
return archive
@ -98,6 +106,17 @@ def get_archive(name):
return None
def reraise_known_error(err):
"""Look whether the caught error is known and re-raise it accordingly"""
caught_error = str(err)
for known_error in KNOWN_ERRORS:
for error in known_error["errors"]:
if error in caught_error:
raise known_error["raise_as"](known_error["message"])
else:
raise err
def test_connection(repository, access_params=None):
"""
Test connecting to a local or remote borg repository.
@ -107,13 +126,7 @@ def test_connection(repository, access_params=None):
try:
return get_info(repository, access_params)
except ActionError as err:
caught_error = str(err)
for known_error in KNOWN_ERRORS:
for error in known_error["errors"]:
if error in caught_error:
raise known_error["raise_as"](known_error["message"])
else:
raise
reraise_known_error(err)
def _backup_handler(packet):
@ -137,6 +150,7 @@ def _backup_handler(packet):
paths.append(manifest_path)
actions.superuser_run(
'backups', ['create-archive', '--name', packet.label, '--paths'] +
# TODO: add ssh_keyfile
paths)
@ -177,8 +191,9 @@ def _restore_exported_archive_handler(packet):
"""Perform restore operation on packet."""
locations = {'directories': packet.directories, 'files': packet.files}
locations_data = json.dumps(locations)
actions.superuser_run('backups', ['restore-exported-archive',
'--path', packet.label], input=locations_data.encode())
actions.superuser_run('backups', ['restore-exported-archive', '--path',
packet.label],
input=locations_data.encode())
def _restore_archive_handler(packet):
@ -186,7 +201,8 @@ def _restore_archive_handler(packet):
locations = {'directories': packet.directories, 'files': packet.files}
locations_data = json.dumps(locations)
actions.superuser_run('backups', ['restore-archive', '--path',
packet.label, '--destination', '/'], input=locations_data.encode())
packet.label, '--destination', '/'],
input=locations_data.encode())
def restore_from_upload(path, apps=None):

View File

@ -84,7 +84,7 @@ class UploadForm(forms.Form):
help_text=_('Select the backup file you want to upload'))
class CreateRepositoryForm(forms.Form):
class AddRepositoryForm(forms.Form):
repository = forms.CharField(
label=_('SSH Repository Path'), strip=True,
help_text=_('Path of the new repository. Example: '
@ -125,7 +125,7 @@ class CreateRepositoryForm(forms.Form):
)
def clean(self):
cleaned_data = super(CreateRepositoryForm, self).clean()
cleaned_data = super(AddRepositoryForm, self).clean()
passphrase = cleaned_data.get("encryption_passphrase")
confirm_passphrase = cleaned_data.get("confirm_encryption_passphrase")

View File

@ -0,0 +1,180 @@
#
# This file is part of FreedomBox.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
"""
Manage remote storage locations
"""
import json
import logging
import os
from uuid import uuid1
from django.utils.translation import ugettext as _
from plinth import kvstore
from plinth.errors import ActionError
from . import sshfs, list_archives, reraise_known_error, REMOTE_LOCATIONS_KEY
from .errors import BorgError
logger = logging.getLogger(__name__)
MOUNTPOINT = '/media/'
def add(path, repotype, encryption, access_params, store_passwords, added_by):
locations = get_locations()
location = {
'uuid': str(uuid1()),
'path': path,
'type': repotype,
'encryption': encryption,
'added_by': added_by
}
if store_passwords:
if 'encryption_passphrase' in access_params:
location['encryption_passphrase'] = \
access_params['encryption_passphrase']
if 'ssh_password' in access_params:
location['ssh_password'] = access_params['ssh_password']
locations.append(location)
kvstore.set(REMOTE_LOCATIONS_KEY, json.dumps(locations))
def delete(uuid):
"""Umount a location, remove it from kvstore and unlink the mountpoint"""
locations = get_locations()
location = get_location(uuid)
mountpoint = os.path.join(MOUNTPOINT, location['uuid'])
locations = list(filter(lambda location: location['uuid'] != uuid,
locations))
kvstore.set(REMOTE_LOCATIONS_KEY, json.dumps(locations))
if os.path.exists(mountpoint):
try:
sshfs.umount(mountpoint)
except ActionError:
pass
try:
os.unlink(mountpoint)
except Exception as err:
logger.error(err)
def get_archives(uuid=None):
"""
Get archives of one or all locations.
returns: {
uuid: {
'path': path,
'type': type,
'archives': [],
'error': error_message
}
}
"""
locations = {}
for location in get_locations():
mountpoint = os.path.join(MOUNTPOINT, location['uuid'])
new_location = {
'path': location['path'],
'mounted': uuid_is_mounted(location['uuid']),
}
if new_location['mounted']:
try:
new_location['archives'] = list_archives(mountpoint)
except BorgError as err:
new_location['error'] = str(err)
except Exception as err:
logger.error(err)
new_location['error'] = _("Access failed")
locations[location['uuid']] = new_location
return locations
def get_locations(location_type=None):
"""Get list of all locations"""
# TODO: hold locations in memory?
locations = kvstore.get_default(REMOTE_LOCATIONS_KEY, [])
if locations:
locations = json.loads(locations)
if location_type:
locations = [location for location in locations if 'type' in location
and location['type'] == location_type]
return locations
def get_location(uuid):
locations = get_locations()
return list(filter(lambda location: location['uuid'] == uuid,
locations))[0]
def _mount_locations(uuid=None):
locations = get_locations(location_type='ssh')
for location in locations:
_mount_location(location)
def _mount_location(location):
# TODO: shouldn't I just store and query the access_params as they are?
# but encryption_passphrase is not an ssh access_param..
mountpoint = os.path.join(MOUNTPOINT, location['uuid'])
is_mounted = False
if sshfs.is_mounted(mountpoint):
is_mounted = True
else:
access_params = _get_access_params(location)
# TODO: use actual feedback of sshfs.mount
try:
sshfs.mount(location['path'], mountpoint, access_params)
except Exception as err:
reraise_known_error(err)
is_mounted = True
return is_mounted
def _umount_location(location):
mountpoint = os.path.join(MOUNTPOINT, location['uuid'])
return sshfs.umount(mountpoint)
def _get_access_params(location):
keys = ['encryption_passphrase', 'ssh_keyfile', 'ssh_password']
access_params = {key: location[key] for key in keys if key in location}
if location['type'] == 'ssh':
if 'ssh_keyfile' not in location and 'ssh_password' not in \
location:
raise ValueError('Missing credentials')
return access_params
def mount_uuid(uuid):
location = get_location(uuid)
mounted = False
if location:
mounted = _mount_location(location)
return mounted
def umount_uuid(uuid):
location = get_location(uuid)
return _umount_location(location)
def uuid_is_mounted(uuid):
mountpoint = os.path.join(MOUNTPOINT, uuid)
return sshfs.is_mounted(mountpoint)

View File

@ -28,6 +28,17 @@
.share-operations {
text-align: right;
}
.mount-error {
padding: 0px 10px 0px 10px;
color: orange;
}
.mount-success {
padding: 0px 10px 0px 10px;
color: black;
}
.inline-block {
display: inline-block;
}
</style>
{% endblock %}
@ -48,7 +59,7 @@
id="archives-list">
<thead>
<tr>
<th>{% trans "Name" %}</th>
<th>{{ box_name }} storage</th>
<th></th>
</tr>
</thead>
@ -78,4 +89,16 @@
</table>
{% endif %}
{% for uuid,location in remote_archives.items %}
{% include "backups_location.inc" %}
{% endfor %}
<br />
<a title="{% trans 'Create new repository' %}"
role="button" class="btn btn-primary"
href="{% url 'backups:location-add' %}">
{% trans '+ Add Remote Repository' %}
</a>
{% endblock %}

View File

@ -34,7 +34,7 @@
value="{% trans "Submit" %}"/>
<input type="submit" class="btn btn-secondary" value="Test Connection"
title="{% trans 'Test Connection to Repository' %}"
formaction="{% url 'backups:test-repository' %}" />
formaction="{% url 'backups:location-test' %}" />
</form>
{% endblock %}

View File

@ -0,0 +1,83 @@
{% comment %}
#
# This file is part of FreedomBox.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
{% endcomment %}
{% load i18n %}
<table class="table table-bordered table-condensed table-striped"
id="archives-list">
<thead>
<tr>
<th>
{{ location.path }}
{% if location.mounted %}
<form action="{% url 'backups:location-umount' uuid %}" method="POST"
class="inline-block" >
{% csrf_token %}
<button type="submit" class="btn btn-sm btn-default"
title="{% trans 'Unount Location' %}">
<span class="glyphicon glyphicon-eject" aria-hidden="true">
</button>
</form>
{% else %}
<!-- With GET redirects the browser URL would be pointing to the
redirected page - use POST instead.
-->
<form action="{% url 'backups:location-mount' uuid %}" method="POST"
class="inline-block" >
{% csrf_token %}
<button type="submit" class="btn btn-sm btn-default"
title="{% trans 'Mount Location' %}">
<span class="glyphicon glyphicon-eye-open" aria-hidden="true">
</button>
</form>
{% endif %}
<a title="{% trans 'Remove Location. This will not delete the remote backup.' %}"
role="button" class="location-remove btn btn-sm btn-default"
href="{% url 'backups:location-remove' uuid %}">
<span class="glyphicon glyphicon-trash" aria-hidden="true">
</a>
</th>
<th>
</th>
</tr>
</thead>
<tbody>
{% for archive in location.archives %}
<tr id="archive-{{ archive.name }}" class="archive">
<td class="archive-name">{{ archive.name }}</td>
<td class="archive-operations">
<a class="archive-export btn btn-sm btn-default" target="_blank"
href="{% url 'backups:export-and-download' archive.name %}">
{% trans "Download" %}
</a>
<a class="archive-export btn btn-sm btn-default"
href="{% url 'backups:restore-archive' archive.name %}">
{% trans "Restore" %}
</a>
<a class="archive-delete btn btn-sm btn-default"
href="{% url 'backups:delete' archive.name %}">
<span class="glyphicon glyphicon-trash" aria-hidden="true">
</span>
</a>
</td>
</tr>
{% endfor %}
</tbody>
</table>

View File

@ -18,22 +18,39 @@
#
{% endcomment %}
{% load bootstrap %}
{% load i18n %}
{% block content %}
<h3>{% trans 'Backup repositories' %}</h3>
<h2>{{ title }}</h2>
<p>
{% for repository in repositories %}
Repository: {{ repository.repository }}<br />
{% endfor %}
<b>
{% trans "Are you sure that you want to remove the repository" %}
{{ location.path }}?
</b>
</p>
{% blocktrans %}
The remote repository will not be deleted.
This just removes the repository from the listing on the backup page, you
can add it again later on.
{% endblocktrans %}
</p>
<a title="{% trans 'Create new repository' %}"
role="button" class="btn btn-primary"
href="{% url 'backups:create-repository' %}">
{% trans 'Add Repository' %}
</a>
<p>
<form class="form" method="post">
{% csrf_token %}
<input type="submit" class="btn btn-danger"
value="{% blocktrans trimmed with path=location.path %}
Remove Repository
{% endblocktrans %}"/>
<a class="abort btn btn-sm btn-default"
href="{% url 'backups:index' %}">
{% trans "Abort" %}
</a>
</form>
</p>
{% endblock %}

View File

@ -20,10 +20,10 @@ URLs for the backups module.
from django.conf.urls import url
from .views import IndexView, CreateArchiveView, CreateRepositoryView, \
DeleteArchiveView, UploadArchiveView, ExportAndDownloadView, \
RepositoriesView, RestoreArchiveView, RestoreFromUploadView, \
TestRepositoryView
from .views import IndexView, CreateArchiveView, AddLocationView, \
DeleteArchiveView, ExportAndDownloadView, RemoveLocationView, \
mount_location, umount_location, UploadArchiveView, \
RestoreArchiveView, RestoreFromUploadView, TestLocationView
urlpatterns = [
url(r'^sys/backups/$', IndexView.as_view(), name='index'),
@ -37,10 +37,14 @@ urlpatterns = [
RestoreArchiveView.as_view(), name='restore-archive'),
url(r'^sys/backups/restore-from-upload/$',
RestoreFromUploadView.as_view(), name='restore-from-upload'),
url(r'^sys/backups/repositories/$',
RepositoriesView.as_view(), name='repositories'),
url(r'^sys/backups/repositories/create/$',
CreateRepositoryView.as_view(), name='create-repository'),
url(r'^sys/backups/repositories/test/$',
TestRepositoryView.as_view(), name='test-repository'),
url(r'^sys/backups/locations/add$',
AddLocationView.as_view(), name='location-add'),
url(r'^sys/backups/locations/test/$',
TestLocationView.as_view(), name='location-test'),
url(r'^sys/backups/locations/delete/(?P<uuid>[^/]+)/$',
RemoveLocationView.as_view(), name='location-remove'),
url(r'^sys/backups/locations/mount/(?P<uuid>[^/]+)/$',
mount_location, name='location-mount'),
url(r'^sys/backups/locations/umount/(?P<uuid>[^/]+)/$',
umount_location, name='location-umount'),
]

View File

@ -20,13 +20,11 @@ Views for the backups app.
from datetime import datetime
import gzip
import json
from io import BytesIO
import logging
import mimetypes
import os
import tempfile
from uuid import uuid1
from urllib.parse import unquote
from django.contrib import messages
@ -39,11 +37,11 @@ from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_lazy
from django.views.generic import View, FormView, TemplateView
from plinth import actions, kvstore
from plinth.errors import PlinthError
from plinth import actions
from plinth.errors import PlinthError, ActionError
from plinth.modules import backups, storage
from . import api, forms, SESSION_PATH_VARIABLE, REPOSITORY
from . import api, forms, SESSION_PATH_VARIABLE, REPOSITORY, remote_locations
from .decorators import delete_tmp_backup_file
from .errors import BorgRepositoryDoesNotExistError
@ -58,9 +56,6 @@ subsubmenu = [{
}, {
'url': reverse_lazy('backups:create'),
'text': ugettext_lazy('Create')
}, {
'url': reverse_lazy('backups:repositories'),
'text': ugettext_lazy('Repositories')
}]
@ -69,6 +64,9 @@ class IndexView(TemplateView):
"""View to show list of archives."""
template_name = 'backups.html'
def get_remote_archives(self):
return {} # uuid --> archive list
def get_context_data(self, **kwargs):
"""Return additional context for rendering the template."""
context = super().get_context_data(**kwargs)
@ -76,9 +74,8 @@ class IndexView(TemplateView):
context['description'] = backups.description
context['info'] = backups.get_info(REPOSITORY)
context['archives'] = backups.list_archives(REPOSITORY)
context['remote_archives'] = remote_locations.get_archives()
context['subsubmenu'] = subsubmenu
apps = api.get_all_apps_for_backup()
context['available_apps'] = [app.name for app in apps]
return context
@ -129,7 +126,7 @@ class DeleteArchiveView(SuccessMessageMixin, TemplateView):
"""Delete the archive."""
backups.delete_archive(name)
messages.success(request, _('Archive deleted.'))
return redirect(reverse_lazy('backups:index'))
return redirect('backups:index')
def _get_file_response(path, filename):
@ -292,29 +289,13 @@ class ExportAndDownloadView(View):
return response
class RepositoriesView(TemplateView):
"""View list of repositories."""
template_name = 'backups_repositories.html'
def get_context_data(self, **kwargs):
"""Return additional context for rendering the template."""
context = super().get_context_data(**kwargs)
context['title'] = 'Backup repositories'
# TODO: rename backups_repositories to something more generic,
# that can be used/managed by the storage module too
repositories = kvstore.get_default('backups_repositories', [])
context['repositories'] = json.loads(repositories)
context['subsubmenu'] = subsubmenu
return context
class CreateRepositoryView(SuccessMessageMixin, FormView):
"""View to create a new repository."""
form_class = forms.CreateRepositoryForm
class AddLocationView(SuccessMessageMixin, FormView):
"""View to create a new remote backup location."""
form_class = forms.AddRepositoryForm
prefix = 'backups'
template_name = 'backups_create_repository.html'
success_url = reverse_lazy('backups:repositories')
success_message = _('Created new repository.')
template_name = 'backups_add_location.html'
success_url = reverse_lazy('backups:index')
success_message = _('Added new location.')
def get_context_data(self, **kwargs):
"""Return additional context for rendering the template."""
@ -329,48 +310,83 @@ class CreateRepositoryView(SuccessMessageMixin, FormView):
encryption = form.cleaned_data['encryption']
encryption_passphrase = form.cleaned_data['encryption_passphrase']
ssh_password = form.cleaned_data['ssh_password']
ssh_keyfile = form.cleaned_data['ssh_keyfile']
store_passwords = form.cleaned_data['store_passwords']
# TODO: add ssh_keyfile
# ssh_keyfile = form.cleaned_data['ssh_keyfile']
# TODO: create borg repository if it doesn't exist
import pdb; pdb.set_trace()
access_params = {}
if encryption_passphrase:
access_params['encryption_passphrase'] = encryption_passphrase
if ssh_password:
access_params['ssh_password'] = ssh_password
"""
if ssh_keyfile:
access_params['ssh_keyfile'] = ssh_keyfile
"""
remote_locations.add(repository, 'ssh', encryption, access_params,
store_passwords, 'backups')
# Create the borg repository if it doesn't exist
try:
backups.test_connection(repository, ssh_password)
backups.test_connection(repository, access_params)
except BorgRepositoryDoesNotExistError:
access_params = {}
if encryption_passphrase:
access_params['encryption_passphrase'] = encryption_passphrase
if ssh_keyfile:
access_params['ssh_keyfile'] = ssh_keyfile
backups.create_repository(repository, encryption,
access_params=access_params)
repositories = kvstore.get_default('backups_repositories', [])
if repositories:
repositories = json.loads(repositories)
new_repo = {
'uuid': str(uuid1()),
'repository': repository,
'encryption': encryption,
}
if form.cleaned_data['store_passwords']:
new_repo['encryption_passphrase'] = \
form.cleaned_data['encryption_passphrase']
repositories.append(new_repo)
kvstore.set('backups_repositories', json.dumps(repositories))
return super().form_valid(form)
class TestRepositoryView(TemplateView):
class TestLocationView(TemplateView):
"""View to create a new repository."""
template_name = 'backups_test_repository.html'
template_name = 'backups_test_location.html'
def post(self, request):
# TODO: add support for borg encryption
# TODO: add support for borg encryption and ssh keyfile
context = self.get_context_data()
repository = request.POST['backups-repository']
ssh_password = request.POST['backups-ssh_password']
(error, message) = backups.test_connection(repository, ssh_password)
context["message"] = message
context["error"] = error
access_params = {
'ssh_password': request.POST['backups-ssh_password'],
}
try:
repo_info = backups.test_connection(repository, access_params)
context["message"] = repo_info
except ActionError as err:
context["error"] = str(err)
return self.render_to_response(context)
class RemoveLocationView(SuccessMessageMixin, TemplateView):
"""View to delete an archive."""
template_name = 'backups_remove_repository.html'
def get_context_data(self, uuid, **kwargs):
"""Return additional context for rendering the template."""
context = super().get_context_data(**kwargs)
context['title'] = _('Remove Repository')
context['location'] = remote_locations.get_location(uuid)
return context
def post(self, request, uuid):
"""Delete the archive."""
remote_locations.delete(uuid)
messages.success(request, _('Repository removed. The remote backup '
'itself was not deleted.'))
return redirect('backups:index')
def umount_location(request, uuid):
remote_locations.umount_uuid(uuid)
if remote_locations.uuid_is_mounted(uuid):
messages.error(request, _('Unmounting failed!'))
return redirect('backups:index')
def mount_location(request, uuid):
try:
remote_locations.mount_uuid(uuid)
except Exception as err:
msg = "%s: %s" % (_('Mounting failed'), str(err))
messages.error(request, msg)
else:
if not remote_locations.uuid_is_mounted(uuid):
messages.error(request, _('Mounting failed'))
return redirect('backups:index')