nik/FreedomBox
1
0
Fork 0
mirror of https://github.com/freedombox/FreedomBox.git synced 2026-01-21 07:55:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Backups, remote repositories: implement init, info and some test

Browse Source 
- added functionality to use remote repositories
- added some tests

Reviewed-by: James Valleroy <jvalleroy@mailbox.org>
This commit is contained in:
Michael Pimmer 2018-11-27 03:03:58 +00:00 committed by James Valleroy
parent 4eeceaa695
commit 0e2489ec23
No known key found for this signature in database
GPG Key ID: 77C0C75E7B650808
6 changed files with 289 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
actions/backups
View File

@ -17,7 +17,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
"""
Configuration helper for backups.
Wrapper to handle backups using borg-backups.
"""
import argparse
@ -41,6 +41,10 @@ def parse_arguments():
'setup', help='Create repository if it does not already exist')
info = subparsers.add_parser('info', help='Show repository information')
info.add_argument('--repository', help='Repository path', required=True)
init = subparsers.add_parser('init', help='Initialize a repository')
init.add_argument('--repository', help='Repository path', required=True)
init.add_argument('--encryption', help='Enryption of the repository',
required=True)
subparsers.add_parser('list', help='List repository contents')
create = subparsers.add_parser('create', help='Create archive')
@ -51,10 +55,17 @@ def parse_arguments():
delete = subparsers.add_parser('delete', help='Delete archive')
delete.add_argument('--name', help='Archive name', required=True)
export_help='Export archive contents as tar on stdout'
export_help = 'Export archive contents as tar on stdout'
export_tar = subparsers.add_parser('export-tar', help=export_help)
export_tar.add_argument('--name', help='Archive name)',
required=True)
required=True)
# TODO: add parameters to missing commands (list, create, delete, export)
for cmd in [info, init]:
cmd.add_argument('--ssh-keyfile', help='Path of private ssh key',
default=None)
cmd.add_argument('--encryption-passphrase',
help='Encryption passphrase',
default=None)
get_exported_archive_apps = subparsers.add_parser(
'get-exported-archive-apps',
@ -71,14 +82,14 @@ def parse_arguments():
restore_exported_archive = subparsers.add_parser(
'restore-exported-archive',
help='Restore files from an exported archive')
restore_exported_archive.add_argument('--path',
help='Tarball file path', required=True)
restore_exported_archive.add_argument('--path', help='Tarball file path',
required=True)
restore_archive = subparsers.add_parser(
'restore-archive', help='Restore files from an archive')
restore_archive.add_argument('--path', help='Archive path', required=True)
restore_archive.add_argument('--destination', help='Destination',
required=True)
required=True)
subparsers.required = True
return parser.parse_args()
@ -93,34 +104,65 @@ def subcommand_setup(_):
if not os.path.exists(path):
os.makedirs(path)
subprocess.run(['borg', 'init', '--encryption', 'none', REPOSITORY])
run(['borg', 'init', '--encryption', 'none', REPOSITORY])
def init(repository, encryption, env=None):
"""Initialize a local or remote borg repository"""
# TODO: verify that the repository does not exist?
# TODO: does remote borg also create folders if they don't exist?
if encryption != 'none' and 'BORG_PASSPHRASE' not in env:
raise ValueError('No encryption passphrase provided')
cmd = ['borg', 'init', '--encryption', encryption, repository]
run(cmd, env=env)
def get_env(arguments):
"""Create encryption and ssh kwargs out of given arguments"""
env = dict(os.environ, BORG_RELOCATED_REPO_ACCESS_IS_OK='yes')
if arguments.encryption_passphrase:
env['BORG_PASSPHRASE'] = arguments.encryption_passphrase
if arguments.ssh_keyfile:
env['BORG_RSH'] = "ssh -i %s" % arguments.ssh_keyfile
else:
password = read_password()
if password:
env['SSHPASS'] = password
env['BORG_RSH'] = 'sshpass -e ssh -o StrictHostKeyChecking=no'
return env
def subcommand_init(arguments):
env = get_env(arguments)
init(arguments.repository, arguments.encryption, env=env)
def subcommand_info(arguments):
"""Show repository information."""
run(['borg', 'info', '--json', arguments.repository])
env = get_env(arguments)
cmd = ['borg', 'info', '--json', arguments.repository]
run(cmd, env=env)
def subcommand_list(_):
"""List repository contents."""
subprocess.run(['borg', 'list', '--json', REPOSITORY], check=True)
run(['borg', 'list', '--json', REPOSITORY])
def subcommand_create(arguments):
"""Create archive."""
paths = filter(os.path.exists, arguments.paths)
subprocess.run([
run([
'borg',
'create',
'--json',
REPOSITORY + '::' + arguments.name,
] + list(paths), check=True)
] + list(paths))
def subcommand_delete(arguments):
"""Delete archive."""
subprocess.run(['borg', 'delete', REPOSITORY + '::' + arguments.name],
check=True)
run(['borg', 'delete', REPOSITORY + '::' + arguments.name])
def _extract(archive_path, destination, locations=None):
@ -150,9 +192,7 @@ def _extract(archive_path, destination, locations=None):
def subcommand_export_tar(arguments):
"""Export archive contents as tar stream on stdout."""
subprocess.run([
'borg', 'export-tar', REPOSITORY + '::' + arguments.name, '-'],
check=True)
run(['borg', 'export-tar', REPOSITORY + '::' + arguments.name, '-'])
def _read_archive_file(archive, filepath):
@ -165,7 +205,7 @@ def subcommand_get_archive_apps(arguments):
"""Get list of apps included in archive."""
manifest_folder = os.path.relpath(MANIFESTS_FOLDER, '/')
borg_call = ['borg', 'list', arguments.path, manifest_folder,
'--format', '{path}{NEWLINE}']
'--format', '{path}{NEWLINE}']
try:
manifest_path = subprocess.check_output(borg_call).decode().strip()
except subprocess.CalledProcessError:
@ -173,8 +213,7 @@ def subcommand_get_archive_apps(arguments):
manifest = None
if manifest_path:
manifest_data = _read_archive_file(arguments.path,
manifest_path)
manifest_data = _read_archive_file(arguments.path, manifest_path)
manifest = json.loads(manifest_data)
if manifest:
for app in _get_apps_of_manifest(manifest):
@ -246,17 +285,12 @@ def read_password():
return ''.join(sys.stdin)
def run(cmd):
"""Pass provided passwords on to borg"""
env = dict(os.environ, BORG_RELOCATED_REPO_ACCESS_IS_OK='yes')
password = read_password()
def run(cmd, env=None):
"""Wrap the command with ssh password or keyfile authentication"""
# If the remote server asks for a password but no password is
# provided, we get stuck at asking the password.
timeout = None
if password:
env['SSHPASS'] = password
env['BORG_RSH'] = 'sshpass -e ssh -o StrictHostKeyChecking=no'
else:
# When no password is given ssh might ask for a password and get stuck.
# Use timeout to abort early.
if 'BORG_RSH' in env and 'SSHPASS' not in env:
timeout = TIMEOUT
subprocess.run(cmd, check=True, env=env, timeout=timeout)

76
plinth/modules/backups/__init__.py
View File

@ -26,6 +26,8 @@ from django.utils.translation import ugettext_lazy as _
from plinth import actions
from plinth.menu import main_menu
from plinth.errors import ActionError
from .errors import BorgError, BorgRepositoryDoesNotExistError
from . import api
@ -45,6 +47,23 @@ MANIFESTS_FOLDER = '/var/lib/plinth/backups-manifests/'
REPOSITORY = '/var/lib/freedombox/borgbackup'
# session variable name that stores when a backup file should be deleted
SESSION_PATH_VARIABLE = 'fbx-backups-upload-path'
# known errors that come up when remotely accessing a borg repository
# 'errors' are error strings to look for in the stacktrace.
KNOWN_ERRORS = [{
"errors": ["subprocess.TimeoutExpired"],
"message": _("Server not reachable - try providing a password."),
"raise_as": BorgError,
},
{
"errors": ["Connection refused"],
"message": _("Connection refused"),
"raise_as": BorgError,
},
{
"errors": ["not a valid repository", "does not exist"],
"message": _("Connection works - Repository does not exist"),
"raise_as": BorgRepositoryDoesNotExistError,
}]
def init():
@ -59,12 +78,17 @@ def setup(helper, old_version=None):
helper.call('post', actions.superuser_run, 'backups', ['setup'])
def get_info(repository, password=None):
args = ['backups', ['info', '--repository', repository]]
def get_info(repository, encryption_passphrase=None, ssh_password=None,
ssh_keyfile=None):
args = ['info', '--repository', repository]
kwargs = {}
if password is not None:
kwargs['input'] = password.encode()
output = actions.superuser_run(*args, **kwargs)
if ssh_password is not None:
kwargs['input'] = ssh_password.encode()
if ssh_keyfile is not None:
args += ['--ssh-keyfile', ssh_keyfile]
if encryption_passphrase is not None:
args += ['--encryption-passphrase', encryption_passphrase]
output = actions.superuser_run('backups', args, **kwargs)
return json.loads(output)
@ -81,6 +105,30 @@ def get_archive(name):
return None
def test_connection(repository, encryption_passphrase=None, ssh_password=None,
ssh_keyfile=None):
"""
Test connecting to a local or remote borg repository.
Tries to detect (and throw) some known ssh or borg errors.
Returns 'borg info' information otherwise.
"""
try:
# TODO: instead of passing encryption_passphrase, ssh_password and
# ssh_keyfile around all the time, try using an 'options' dict.
message = get_info(repository,
encryption_passphrase=encryption_passphrase,
ssh_password=ssh_password, ssh_keyfile=ssh_keyfile)
return message
except ActionError as err:
caught_error = str(err)
for known_error in KNOWN_ERRORS:
for error in known_error["errors"]:
if error in caught_error:
raise known_error["raise_as"](known_error["message"])
else:
raise
def _backup_handler(packet):
"""Performs backup operation on packet."""
if not os.path.exists(MANIFESTS_FOLDER):
@ -108,6 +156,24 @@ def create_archive(name, app_names):
api.backup_apps(_backup_handler, app_names, name)
def create_repository(repository, encryption, encryption_passphrase=None,
ssh_keyfile=None, ssh_password=None):
cmd = ['init', '--repository', repository, '--encryption', encryption]
if ssh_keyfile:
cmd += ['--ssh-keyfile', ssh_keyfile]
if encryption_passphrase:
cmd += ['--encryption-passphrase', encryption_passphrase]
kwargs = {}
if ssh_password:
kwargs['input'] = ssh_password.encode()
output = actions.superuser_run('backups', cmd, **kwargs)
if output:
output = json.loads(output)
return output
def delete_archive(name):
actions.superuser_run('backups', ['delete', '--name', name])

28
plinth/modules/backups/errors.py Normal file
View File

@ -0,0 +1,28 @@
#
# This file is part of FreedomBox.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from plinth.errors import PlinthError
class BorgError(PlinthError):
"""Generic class for borg errors that """
pass
class BorgRepositoryDoesNotExistError(BorgError):
"""Borg access to a repository works but the repository does not exist"""
pass

29
plinth/modules/backups/forms.py
View File

@ -88,13 +88,12 @@ class CreateRepositoryForm(forms.Form):
repository = forms.CharField(
label=_('SSH Repository Path'), strip=True,
help_text=_('Path of the new repository. Example: '
'<i>user@host/path/to/repo/</i>'))
'<i>user@host:~/path/to/repo/</i>'))
ssh_password = forms.CharField(
label=_('SSH password'), strip=True,
help_text=_('Password of the SSH Server.<br />'
'If you have set up <a href="https://www.ssh.com/ssh/key/"'
'target="_blank">SSH key-based authentication</a> you can '
'omit the password.'),
label=_('SSH server password'), strip=True,
help_text=_('Password of the SSH Server.<br />'
'SSH key-based authentication is not yet possible.'),
widget=forms.PasswordInput(),
required=False)
encryption = forms.ChoiceField(
label=_('Encryption'),
@ -104,33 +103,33 @@ class CreateRepositoryForm(forms.Form):
'<b>You need this password to restore a backup!</b>')),
choices=[('repokey', 'Key in Repository'), ('none', 'None')]
)
passphrase = forms.CharField(
encryption_passphrase = forms.CharField(
label=_('Passphrase'),
help_text=_('Passphrase; Only needed when using encryption.'),
widget=forms.PasswordInput(),
required=False
)
confirm_passphrase = forms.CharField(
confirm_encryption_passphrase = forms.CharField(
label=_('Confirm Passphrase'),
help_text=_('Repeat the passphrase.'),
widget=forms.PasswordInput(),
required=False
)
store_passphrase = forms.BooleanField(
label=_('Store passphrase on FreedomBox'),
store_passwords = forms.BooleanField(
label=_('Store passwords on FreedomBox'),
help_text=format_lazy(
_('Store the passphrase on your {box_name}.'
'<br />You need to store the passphrase if you want to run '
_('Store the passwords on your {box_name}.'
'<br />You need to store passwords if you want to run '
'recurrent backups.'), box_name=_(cfg.box_name)),
required=False
)
def clean(self):
cleaned_data = super(CreateRepositoryForm, self).clean()
passphrase = cleaned_data.get("passphrase")
confirm_passphrase = cleaned_data.get("confirm_passphrase")
passphrase = cleaned_data.get("encryption_passphrase")
confirm_passphrase = cleaned_data.get("confirm_encryption_passphrase")
if passphrase != confirm_passphrase:
raise forms.ValidationError(
"passphrase and confirm_passphrase do not match"
"The entered encryption passphrases do not match"
)

46
plinth/modules/backups/views.py
View File

@ -40,11 +40,12 @@ from django.utils.translation import ugettext_lazy
from django.views.generic import View, FormView, TemplateView
from plinth import actions, kvstore
from plinth.errors import PlinthError, ActionError
from plinth.errors import PlinthError
from plinth.modules import backups, storage
from . import api, forms, SESSION_PATH_VARIABLE, REPOSITORY
from .decorators import delete_tmp_backup_file
from .errors import BorgRepositoryDoesNotExistError
logger = logging.getLogger(__name__)
@ -316,22 +317,41 @@ class CreateRepositoryView(SuccessMessageMixin, FormView):
def get_context_data(self, **kwargs):
"""Return additional context for rendering the template."""
context = super().get_context_data(**kwargs)
context['title'] = _('Create new repository')
context['title'] = _('Create remote backup repository')
context['subsubmenu'] = subsubmenu
return context
def form_valid(self, form):
"""Restore files from the archive on valid form submission."""
repository = form.cleaned_data['repository']
encryption = form.cleaned_data['encryption']
encryption_passphrase = form.cleaned_data['encryption_passphrase']
ssh_password = form.cleaned_data['ssh_password']
ssh_keyfile = form.cleaned_data['ssh_keyfile']
# TODO: create borg repository if it doesn't exist
import pdb; pdb.set_trace()
try:
backups.test_connection(repository, ssh_password)
except BorgRepositoryDoesNotExistError:
kwargs = {}
if encryption_passphrase:
kwargs['encryption_passphrase'] = encryption_passphrase
if ssh_keyfile:
kwargs['ssh_keyfile'] = ssh_keyfile
backups.create_repository(repository, encryption, **kwargs)
repositories = kvstore.get_default('backups_repositories', [])
if repositories:
repositories = json.loads(repositories)
new_repo = {
'uuid': str(uuid1()),
'repository': form.cleaned_data['repository'],
'encryption': form.cleaned_data['encryption'],
'repository': repository,
'encryption': encryption,
}
if form.cleaned_data['store_passphrase']:
new_repo['passphrase'] = form.cleaned_data['passphrase']
if form.cleaned_data['store_passwords']:
new_repo['encryption_passphrase'] = \
form.cleaned_data['encryption_passphrase']
repositories.append(new_repo)
kvstore.set('backups_repositories', json.dumps(repositories))
return super().form_valid(form)
@ -342,18 +362,12 @@ class TestRepositoryView(TemplateView):
template_name = 'backups_test_repository.html'
def post(self, request):
# TODO: add support for borg encryption
context = self.get_context_data()
repository = request.POST['backups-repository']
ssh_password = request.POST['backups-ssh_password']
try:
info = backups.get_info(repository, password=ssh_password)
except ActionError as err:
if "subprocess.TimeoutExpired" in str(err):
msg = _("Server not reachable - try providing a password.")
context["error"] = msg
else:
context["error"] = str(err)
else:
context["message"] = info
(error, message) = backups.test_connection(repository, ssh_password)
context["message"] = message
context["error"] = error
return self.render_to_response(context)

83
plinth/tests/test_backups.py Normal file
View File

@ -0,0 +1,83 @@
#
# This file is part of FreedomBox.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
"""
Test the backups action script.
"""
import os
import shutil
import tempfile
import unittest
from plinth import cfg
from plinth.modules import backups
euid = os.geteuid()
class TestBackups(unittest.TestCase):
"""Test creating, reading and deleting a repository"""
@classmethod
def setUpClass(cls):
"""Initial setup for all the classes."""
cls.action_directory = tempfile.TemporaryDirectory()
cls.backup_directory = tempfile.TemporaryDirectory()
cfg.actions_dir = cls.action_directory.name
actions_dir = os.path.join(os.path.dirname(__file__), '..', '..',
'actions')
shutil.copy(os.path.join(actions_dir, 'backups'), cfg.actions_dir)
@classmethod
def tearDownClass(cls):
"""Cleanup after all the tests are completed."""
cls.action_directory.cleanup()
cls.backup_directory.cleanup()
@unittest.skipUnless(euid == 0, 'Needs to be root')
def test_nonexisting_repository(self):
nonexisting_dir = os.path.join(self.backup_directory.name,
'does_not_exist')
with self.assertRaises(backups.errors.BorgRepositoryDoesNotExistError):
backups.test_connection(nonexisting_dir)
@unittest.skipUnless(euid == 0, 'Needs to be root')
def test_empty_dir(self):
empty_dir = os.path.join(self.backup_directory.name, 'empty_dir')
os.mkdir(empty_dir)
with self.assertRaises(backups.errors.BorgRepositoryDoesNotExistError):
backups.test_connection(empty_dir)
@unittest.skipUnless(euid == 0, 'Needs to be root')
def test_create_unencrypted_repository(self):
repo_path = os.path.join(self.backup_directory.name, 'borgbackup')
backups.create_repository(repo_path, 'none')
info = backups.get_info(repo_path)
assert 'encryption' in info
@unittest.skipUnless(euid == 0, 'Needs to be root')
def test_create_encrypted_repository(self):
repo_path = os.path.join(self.backup_directory.name,
'borgbackup_encrypted')
passphrase = '12345'
# create_repository is supposed to create the folder automatically
# if it does not exist
backups.create_repository(repo_path, 'repokey',
encryption_passphrase=passphrase)
assert backups.get_info(repo_path, encryption_passphrase=passphrase)
assert backups.test_connection(repo_path,
encryption_passphrase=passphrase)