From 01492895c49e561d53938bc15294200097d84d7d Mon Sep 17 00:00:00 2001 From: Sunil Mohan Adapa Date: Wed, 30 Jan 2019 12:29:46 -0800 Subject: [PATCH] backups: Fix incomplete download archives - Downloaded archives can't be fully extracted as tar.gz is incomplete at the end and corrupt. This is due to complete gzip streaming implementation that is does not flush the final bytes of gzip stream. Remove custom implementation and get gzipped stream directly from borg. - Fix mimetype for .tar.gz to application/gzip. Signed-off-by: Sunil Mohan Adapa Reviewed-by: James Valleroy --- actions/backups | 3 +- plinth/modules/backups/repository.py | 7 ++--- plinth/modules/backups/views.py | 3 +- plinth/modules/backups/zipstream.py | 47 ---------------------------- 4 files changed, 7 insertions(+), 53 deletions(-) delete mode 100644 plinth/modules/backups/zipstream.py diff --git a/actions/backups b/actions/backups index 190eb9d87..a544d1c8a 100755 --- a/actions/backups +++ b/actions/backups @@ -174,7 +174,8 @@ def _extract(archive_path, destination, locations=None, env=None): def subcommand_export_tar(arguments): """Export archive contents as tar stream on stdout.""" - run(['borg', 'export-tar', arguments.path, '-'], arguments=arguments) + run(['borg', 'export-tar', arguments.path, '-', '--tar-filter=gzip'], + arguments=arguments) def _read_archive_file(archive, filepath, env=None): diff --git a/plinth/modules/backups/repository.py b/plinth/modules/backups/repository.py index aecc188a8..adba67262 100644 --- a/plinth/modules/backups/repository.py +++ b/plinth/modules/backups/repository.py @@ -29,8 +29,7 @@ from plinth import actions from plinth.errors import ActionError from . import api, network_storage, _backup_handler, ROOT_REPOSITORY_NAME, \ - ROOT_REPOSITORY_UUID, ROOT_REPOSITORY, restore_archive_handler, \ - zipstream + ROOT_REPOSITORY_UUID, ROOT_REPOSITORY, restore_archive_handler from .errors import BorgError, BorgRepositoryDoesNotExistError, SshfsError logger = logging.getLogger(__name__) @@ -141,11 +140,11 @@ class BorgRepository(): def create_repository(self): self.run(['init', '--path', self.repo_path, '--encryption', 'none']) - def get_zipstream(self, archive_name): + def get_download_stream(self, archive_name): args = ['export-tar', '--path', self._get_archive_path(archive_name)] args += self._get_encryption_arguments(self.credentials) proc = self._run('backups', args, run_in_background=True) - return zipstream.ZipStream(proc.stdout, 'readline') + return proc.stdout def get_archive(self, name): for archive in self.list_archives(): diff --git a/plinth/modules/backups/views.py b/plinth/modules/backups/views.py index 7de6c1600..edaf002f0 100644 --- a/plinth/modules/backups/views.py +++ b/plinth/modules/backups/views.py @@ -239,7 +239,8 @@ class DownloadArchiveView(View): filename = '%s.tar.gz' % name response = StreamingHttpResponse( - repository.get_zipstream(name), content_type='application/x-gzip') + repository.get_download_stream(name), + content_type='application/gzip') response['Content-Disposition'] = 'attachment; filename="%s"' % \ filename return response diff --git a/plinth/modules/backups/zipstream.py b/plinth/modules/backups/zipstream.py deleted file mode 100644 index 5da61ecaf..000000000 --- a/plinth/modules/backups/zipstream.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# This file is part of FreedomBox. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# - -import gzip -from io import BytesIO - - -class ZipStream(object): - """Zip a stream that yields binary data""" - - def __init__(self, stream, get_chunk_method): - """ - - stream: the input stream - - get_chunk_method: name of the method to get a chunk of the stream - """ - self.stream = stream - self.buffer = BytesIO() - self.zipfile = gzip.GzipFile(mode='wb', fileobj=self.buffer) - self.get_chunk = getattr(self.stream, get_chunk_method) - - def __next__(self): - line = self.get_chunk() - if not len(line): - raise StopIteration - self.zipfile.write(line) - self.zipfile.flush() - zipped = self.buffer.getvalue() - self.buffer.truncate(0) - self.buffer.seek(0) - return zipped - - def __iter__(self): - return self