Backups: Stream archive downloads/exports

Reviewed-by: James Valleroy <jvalleroy@mailbox.org>

actions/backups

@@ -60,6 +60,11 @@ def parse_arguments():
     export_tar.add_argument('--filepath', help='Destination tarball file path',
                             required=True)
 
+    export_stream = subparsers.add_parser('export-stream',
+                               help='Export archive contents as tar stream')
+    export_stream.add_argument('--name', help='Archive name)',
+                               required=True)
+
     get_exported_archive_apps = subparsers.add_parser(
         'get-exported-archive-apps',
         help='Get list of apps included in exported archive file')
@@ -175,6 +180,13 @@ def subcommand_export_tar(arguments):
         pass
 
 
+def subcommand_export_stream(arguments):
+    """Export archive contents as tar stream."""
+    subprocess.run([
+        'borg', 'export-tar', REPOSITORY + '::' + arguments.name, '-'],
+        check=True)
+
+
 def _read_archive_file(archive, filepath):
     """Read the content of a file inside an archive"""
     arguments = ['borg', 'extract', archive, filepath, '--stdout']

plinth/__main__.py

@@ -308,7 +308,6 @@ def configure_django():
             'plinth.middleware.FirstSetupMiddleware',
             'plinth.modules.first_boot.middleware.FirstBootMiddleware',
             'plinth.middleware.SetupMiddleware',
-            'plinth.modules.backups.middleware.BackupsMiddleware',
         ),
         ROOT_URLCONF='plinth.urls',
         SECURE_BROWSER_XSS_FILTER=True,

plinth/actions.py

@@ -100,35 +100,38 @@ from plinth.errors import ActionError
 LOGGER = logging.getLogger(__name__)
 
 
-def run(action, options=None, input=None, run_in_background=False):
+def run(action, options=None, input=None, run_in_background=False,
+        bufsize=None):
     """Safely run a specific action as the current user.
 
     See actions._run for more information.
     """
-    return _run(action, options, input, run_in_background, False)
+    return _run(action, options, input, run_in_background, False,
+                bufsize=bufsize)
 
 
 def superuser_run(action, options=None, input=None, run_in_background=False,
-                  log_error=True):
+                  bufsize=None, log_error=True):
     """Safely run a specific action as root.
 
     See actions._run for more information.
     """
     return _run(action, options, input, run_in_background, True,
-                log_error=log_error)
+                bufsize=bufsize, log_error=log_error)
 
 
 def run_as_user(action, options=None, input=None, run_in_background=False,
-                become_user=None):
+                bufsize=None, become_user=None):
     """Run a command as a different user.
 
     If become_user is None, run as current user.
     """
-    return _run(action, options, input, run_in_background, False, become_user)
+    return _run(action, options, input, run_in_background, False, become_user,
+                bufsize=bufsize)
 
 
 def _run(action, options=None, input=None, run_in_background=False,
-         run_as_root=False, become_user=None, log_error=True):
+         run_as_root=False, become_user=None, log_error=True, bufsize=None):
     """Safely run a specific action as a normal user or root.
 
     Actions are pulled from the actions directory.
@@ -182,6 +185,10 @@ def _run(action, options=None, input=None, run_in_background=False,
 
     LOGGER.info('Executing command - %s', cmd)
 
+    # Use default bufsize if no bufsize is given.
+    if bufsize is None:
+        bufsize = -1
+
     # Contract 3C: don't interpret shell escape sequences.
     # Contract 5 (and 6-ish).
     kwargs = {
@@ -189,6 +196,7 @@ def _run(action, options=None, input=None, run_in_background=False,
         "stdout": subprocess.PIPE,
         "stderr": subprocess.PIPE,
         "shell": False,
+        "bufsize": bufsize,
     }
     if cfg.develop:
         # In development mode pass on local pythonpath to access Plinth

plinth/modules/backups/__init__.py

@@ -45,8 +45,6 @@ MANIFESTS_FOLDER = '/var/lib/plinth/backups-manifests/'
 REPOSITORY = '/var/lib/freedombox/borgbackup'
 # session variable name that stores when a backup file should be deleted
 SESSION_BACKUP_VARIABLE = 'fbx-backup-filestamp'
-# default backup path for temporary backup files during down- or upload
-UPLOAD_BACKUP_PATH = '/tmp/freedombox-backup.tar.gz'
 
 
 def init():
@@ -108,15 +106,16 @@ def delete_archive(name):
     actions.superuser_run('backups', ['delete', '--name', name])
 
 
-def delete_upload_backup_file():
+def delete_upload_backup_file(path):
-    if os.path.isfile(UPLOAD_BACKUP_PATH):
+    if os.path.isfile(path):
-        os.remove(UPLOAD_BACKUP_PATH)
+        os.remove(path)
 
 
-def export_archive(name, filepath=UPLOAD_BACKUP_PATH):
+def export_archive(name, filepath):
     """Export an archive as .tar.gz file
 
     name: name of the repository (w/o path)
+    filepath: filepath the archive should be exported to
     """
     arguments = ['export-tar', '--name', name, '--filepath', filepath]
     actions.superuser_run('backups', arguments)

plinth/modules/backups/middleware.py

@@ -1,47 +0,0 @@
-#
-# This file is part of FreedomBox.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-"""
-Django middleware to occasionally delete temporary backup files
-"""
-
-import logging
-import random
-import time
-
-from django.utils.deprecation import MiddlewareMixin
-
-from plinth.modules import backups
-
-LOGGER = logging.getLogger(__name__)
-
-
-class BackupsMiddleware(MiddlewareMixin):
-    """Delete outdated backup file."""
-
-    @staticmethod
-    def process_request(request):
-        """Handle a request as Django middleware request handler."""
-        if random.random() > 0.9:
-            if request.session.has_key(backups.SESSION_BACKUP_VARIABLE):
-                now = time.time()
-                if now > request.session[backups.SESSION_BACKUP_VARIABLE]:
-                    backups.delete_upload_backup_file()
-                    del request.session[backups.SESSION_BACKUP_VARIABLE]
-            else:
-                backups.delete_upload_backup_file()
-        return

plinth/modules/backups/views.py

@@ -18,15 +18,17 @@
 Views for the backups app.
 """
 
+import gzip
 import mimetypes
 from datetime import datetime
 import os
+from io import BytesIO
 import time
 from urllib.parse import unquote
 
 from django.contrib import messages
 from django.contrib.messages.views import SuccessMessageMixin
-from django.http import Http404, FileResponse
+from django.http import Http404, FileResponse, StreamingHttpResponse
 from django.shortcuts import redirect
 from django.urls import reverse_lazy
 from django.utils.translation import ugettext as _
@@ -34,9 +36,9 @@ from django.utils.translation import ugettext_lazy
 from django.views.generic import View, FormView, TemplateView
 
 from plinth.modules import backups
+from plinth import actions
 
-from . import api, UPLOAD_BACKUP_PATH, forms, \
+from . import api, forms, SESSION_BACKUP_VARIABLE, delete_upload_backup_file
-        SESSION_BACKUP_VARIABLE, delete_upload_backup_file
 
 # number of seconds an uploaded backup file should be kept/stored
 KEEP_UPLOADED_BACKUP_FOR = 60*10
@@ -129,15 +131,6 @@ def _get_file_response(path, filename):
     return response
 
 
-class ExportAndDownloadView(View):
-    """View to export and download an archive."""
-    def get(self, request, name):
-        name = unquote(name)
-        filename = "%s.tar.gz" % name
-        with create_temporary_backup_file(name) as filepath:
-            return _get_file_response(filepath, filename)
-
-
 class create_temporary_backup_file:
     """Create a temporary backup file that gets deleted after using it"""
     # TODO: try using export-tar with FILE parameter '-' and reading stdout:
@@ -152,7 +145,7 @@ class create_temporary_backup_file:
         return self.path
 
     def __exit__(self, type, value, traceback):
-        delete_upload_backup_file()
+        delete_upload_backup_file(self.path)
 
 
 class UploadArchiveView(SuccessMessageMixin, FormView):
@@ -244,3 +237,46 @@ class RestoreArchiveView(BaseRestoreView):
         archive_path = backups.get_archive_path(self.kwargs['name'])
         backups.restore(archive_path, form.cleaned_data['selected_apps'])
         return super().form_valid(form)
+
+
+class ZipStream(object):
+    """Zip a stream that yields binary data"""
+
+    def __init__(self, stream, get_chunk_method):
+        """
+        - stream: the input stream
+        - get_chunk_method: name of the method that yields chunks
+        """
+        self.stream = stream
+        self.buffer = BytesIO()
+        self.zipfile = gzip.GzipFile(mode='wb', fileobj=self.buffer)
+        self.get_chunk = getattr(self.stream, get_chunk_method)
+
+    def __next__(self):
+        line = self.get_chunk()
+        if not len(line):
+            raise StopIteration
+        self.zipfile.write(line)
+        self.zipfile.flush()
+        zipped = self.buffer.getvalue()
+        self.buffer.truncate(0)
+        self.buffer.seek(0)
+        return zipped
+
+    def __iter__(self):
+        return self
+
+
+class ExportAndDownloadView(View):
+    """View to export and download an archive as stream."""
+    def get(self, request, name):
+        name = unquote(name)
+        filename = "%s.tar.gz" % name
+        args = ['export-stream', '--name', name]
+        proc = actions.superuser_run('backups', args, run_in_background=True,
+                bufsize=1)
+        zipStream = ZipStream(proc.stdout, 'readline')
+        response = StreamingHttpResponse(zipStream,
+                content_type="application/x-gzip")
+        response['Content-Disposition'] = 'attachment; filename="%s"' % filename
+        return response