diff --git a/actions/backups b/actions/backups index 46474abe8..290be033c 100755 --- a/actions/backups +++ b/actions/backups @@ -60,6 +60,11 @@ def parse_arguments(): export_tar.add_argument('--filepath', help='Destination tarball file path', required=True) + export_stream = subparsers.add_parser('export-stream', + help='Export archive contents as tar stream') + export_stream.add_argument('--name', help='Archive name)', + required=True) + get_exported_archive_apps = subparsers.add_parser( 'get-exported-archive-apps', help='Get list of apps included in exported archive file') @@ -175,6 +180,13 @@ def subcommand_export_tar(arguments): pass +def subcommand_export_stream(arguments): + """Export archive contents as tar stream.""" + subprocess.run([ + 'borg', 'export-tar', REPOSITORY + '::' + arguments.name, '-'], + check=True) + + def _read_archive_file(archive, filepath): """Read the content of a file inside an archive""" arguments = ['borg', 'extract', archive, filepath, '--stdout'] diff --git a/plinth/__main__.py b/plinth/__main__.py index cfd8378aa..9531aafa3 100644 --- a/plinth/__main__.py +++ b/plinth/__main__.py @@ -308,7 +308,6 @@ def configure_django(): 'plinth.middleware.FirstSetupMiddleware', 'plinth.modules.first_boot.middleware.FirstBootMiddleware', 'plinth.middleware.SetupMiddleware', - 'plinth.modules.backups.middleware.BackupsMiddleware', ), ROOT_URLCONF='plinth.urls', SECURE_BROWSER_XSS_FILTER=True, diff --git a/plinth/actions.py b/plinth/actions.py index f9d9c909f..b7cebc43f 100644 --- a/plinth/actions.py +++ b/plinth/actions.py @@ -100,35 +100,38 @@ from plinth.errors import ActionError LOGGER = logging.getLogger(__name__) -def run(action, options=None, input=None, run_in_background=False): +def run(action, options=None, input=None, run_in_background=False, + bufsize=None): """Safely run a specific action as the current user. See actions._run for more information. """ - return _run(action, options, input, run_in_background, False) + return _run(action, options, input, run_in_background, False, + bufsize=bufsize) def superuser_run(action, options=None, input=None, run_in_background=False, - log_error=True): + bufsize=None, log_error=True): """Safely run a specific action as root. See actions._run for more information. """ return _run(action, options, input, run_in_background, True, - log_error=log_error) + bufsize=bufsize, log_error=log_error) def run_as_user(action, options=None, input=None, run_in_background=False, - become_user=None): + bufsize=None, become_user=None): """Run a command as a different user. If become_user is None, run as current user. """ - return _run(action, options, input, run_in_background, False, become_user) + return _run(action, options, input, run_in_background, False, become_user, + bufsize=bufsize) def _run(action, options=None, input=None, run_in_background=False, - run_as_root=False, become_user=None, log_error=True): + run_as_root=False, become_user=None, log_error=True, bufsize=None): """Safely run a specific action as a normal user or root. Actions are pulled from the actions directory. @@ -182,6 +185,10 @@ def _run(action, options=None, input=None, run_in_background=False, LOGGER.info('Executing command - %s', cmd) + # Use default bufsize if no bufsize is given. + if bufsize is None: + bufsize = -1 + # Contract 3C: don't interpret shell escape sequences. # Contract 5 (and 6-ish). kwargs = { @@ -189,6 +196,7 @@ def _run(action, options=None, input=None, run_in_background=False, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "shell": False, + "bufsize": bufsize, } if cfg.develop: # In development mode pass on local pythonpath to access Plinth diff --git a/plinth/modules/backups/__init__.py b/plinth/modules/backups/__init__.py index ae250f8a4..4da8b624a 100644 --- a/plinth/modules/backups/__init__.py +++ b/plinth/modules/backups/__init__.py @@ -45,8 +45,6 @@ MANIFESTS_FOLDER = '/var/lib/plinth/backups-manifests/' REPOSITORY = '/var/lib/freedombox/borgbackup' # session variable name that stores when a backup file should be deleted SESSION_BACKUP_VARIABLE = 'fbx-backup-filestamp' -# default backup path for temporary backup files during down- or upload -UPLOAD_BACKUP_PATH = '/tmp/freedombox-backup.tar.gz' def init(): @@ -108,15 +106,16 @@ def delete_archive(name): actions.superuser_run('backups', ['delete', '--name', name]) -def delete_upload_backup_file(): - if os.path.isfile(UPLOAD_BACKUP_PATH): - os.remove(UPLOAD_BACKUP_PATH) +def delete_upload_backup_file(path): + if os.path.isfile(path): + os.remove(path) -def export_archive(name, filepath=UPLOAD_BACKUP_PATH): +def export_archive(name, filepath): """Export an archive as .tar.gz file name: name of the repository (w/o path) + filepath: filepath the archive should be exported to """ arguments = ['export-tar', '--name', name, '--filepath', filepath] actions.superuser_run('backups', arguments) diff --git a/plinth/modules/backups/middleware.py b/plinth/modules/backups/middleware.py deleted file mode 100644 index b9999dc36..000000000 --- a/plinth/modules/backups/middleware.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# This file is part of FreedomBox. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# - -""" -Django middleware to occasionally delete temporary backup files -""" - -import logging -import random -import time - -from django.utils.deprecation import MiddlewareMixin - -from plinth.modules import backups - -LOGGER = logging.getLogger(__name__) - - -class BackupsMiddleware(MiddlewareMixin): - """Delete outdated backup file.""" - - @staticmethod - def process_request(request): - """Handle a request as Django middleware request handler.""" - if random.random() > 0.9: - if request.session.has_key(backups.SESSION_BACKUP_VARIABLE): - now = time.time() - if now > request.session[backups.SESSION_BACKUP_VARIABLE]: - backups.delete_upload_backup_file() - del request.session[backups.SESSION_BACKUP_VARIABLE] - else: - backups.delete_upload_backup_file() - return diff --git a/plinth/modules/backups/views.py b/plinth/modules/backups/views.py index d92187302..976039c8c 100644 --- a/plinth/modules/backups/views.py +++ b/plinth/modules/backups/views.py @@ -18,15 +18,17 @@ Views for the backups app. """ +import gzip import mimetypes from datetime import datetime import os +from io import BytesIO import time from urllib.parse import unquote from django.contrib import messages from django.contrib.messages.views import SuccessMessageMixin -from django.http import Http404, FileResponse +from django.http import Http404, FileResponse, StreamingHttpResponse from django.shortcuts import redirect from django.urls import reverse_lazy from django.utils.translation import ugettext as _ @@ -34,9 +36,9 @@ from django.utils.translation import ugettext_lazy from django.views.generic import View, FormView, TemplateView from plinth.modules import backups +from plinth import actions -from . import api, UPLOAD_BACKUP_PATH, forms, \ - SESSION_BACKUP_VARIABLE, delete_upload_backup_file +from . import api, forms, SESSION_BACKUP_VARIABLE, delete_upload_backup_file # number of seconds an uploaded backup file should be kept/stored KEEP_UPLOADED_BACKUP_FOR = 60*10 @@ -129,15 +131,6 @@ def _get_file_response(path, filename): return response -class ExportAndDownloadView(View): - """View to export and download an archive.""" - def get(self, request, name): - name = unquote(name) - filename = "%s.tar.gz" % name - with create_temporary_backup_file(name) as filepath: - return _get_file_response(filepath, filename) - - class create_temporary_backup_file: """Create a temporary backup file that gets deleted after using it""" # TODO: try using export-tar with FILE parameter '-' and reading stdout: @@ -152,7 +145,7 @@ class create_temporary_backup_file: return self.path def __exit__(self, type, value, traceback): - delete_upload_backup_file() + delete_upload_backup_file(self.path) class UploadArchiveView(SuccessMessageMixin, FormView): @@ -244,3 +237,46 @@ class RestoreArchiveView(BaseRestoreView): archive_path = backups.get_archive_path(self.kwargs['name']) backups.restore(archive_path, form.cleaned_data['selected_apps']) return super().form_valid(form) + + +class ZipStream(object): + """Zip a stream that yields binary data""" + + def __init__(self, stream, get_chunk_method): + """ + - stream: the input stream + - get_chunk_method: name of the method that yields chunks + """ + self.stream = stream + self.buffer = BytesIO() + self.zipfile = gzip.GzipFile(mode='wb', fileobj=self.buffer) + self.get_chunk = getattr(self.stream, get_chunk_method) + + def __next__(self): + line = self.get_chunk() + if not len(line): + raise StopIteration + self.zipfile.write(line) + self.zipfile.flush() + zipped = self.buffer.getvalue() + self.buffer.truncate(0) + self.buffer.seek(0) + return zipped + + def __iter__(self): + return self + + +class ExportAndDownloadView(View): + """View to export and download an archive as stream.""" + def get(self, request, name): + name = unquote(name) + filename = "%s.tar.gz" % name + args = ['export-stream', '--name', name] + proc = actions.superuser_run('backups', args, run_in_background=True, + bufsize=1) + zipStream = ZipStream(proc.stdout, 'readline') + response = StreamingHttpResponse(zipStream, + content_type="application/x-gzip") + response['Content-Disposition'] = 'attachment; filename="%s"' % filename + return response