Backups: Restore directly from archive

Reviewed-by: James Valleroy <jvalleroy@mailbox.org>

actions/backups

@@ -29,7 +29,7 @@ import subprocess
 import sys
 import tarfile
 
-REPOSITORY = '/var/lib/freedombox/borgbackup'
+from plinth.modules.backups import MANIFESTS_FOLDER, REPOSITORY
 
 
 def parse_arguments():
@@ -55,10 +55,10 @@ def parse_arguments():
     extract.add_argument('--destination', help='Extract destination',
                          required=True)
 
-    export = subparsers.add_parser('export',
+    export_tar = subparsers.add_parser('export-tar',
                                    help='Export archive contents as tarball')
-    export.add_argument('--name', help='Archive name', required=True)
+    export_tar.add_argument('--name', help='Archive name', required=True)
-    export.add_argument('--filename', help='Tarball file name', required=True)
+    export_tar.add_argument('--filename', help='Tarball file name', required=True)
 
     list_exports = subparsers.add_parser(
         'list-exports', help='List exported backup archive files')
@@ -71,9 +71,24 @@ def parse_arguments():
     get_export_apps.add_argument(
         '--filename', help='Tarball file name', required=True)
 
-    restore = subparsers.add_parser(
+    get_archive_apps = subparsers.add_parser(
-        'restore', help='Restore files from an exported archive')
+        'get-archive-apps',
-    restore.add_argument('--filename', help='Tarball file name', required=True)
+        help='Get list of apps included in archive')
+    get_archive_apps.add_argument(
+        '--path', help='Path of the archive', required=True)
+
+    restore_exported_archive = subparsers.add_parser(
+        'restore-exported-archive',
+        help='Restore files from an exported archive')
+    # TODO: rename filename to filepath (or just path)
+    restore_exported_archive.add_argument('--filename',
+            help='Tarball file name', required=True)
+
+    restore_archive = subparsers.add_parser(
+        'restore-archive', help='Restore files from an archive')
+    restore_archive.add_argument('--path', help='Archive path', required=True)
+    restore_archive.add_argument('--destination', help='Destination',
+            required=True)
 
     subparsers.required = True
     return parser.parse_args()
@@ -119,21 +134,30 @@ def subcommand_delete(arguments):
 
 
 def subcommand_extract(arguments):
+    """Extract archive contents."""
+    path = REPOSITORY + '::' + arguments.name
+    return _extract(path, arguments.destination)
+
+
+def _extract(archive_path, destination, locations=None):
     """Extract archive contents."""
     prev_dir = os.getcwd()
     env = dict(os.environ, LANG='C.UTF-8')
+    borg_call = ['borg', 'extract', archive_path]
+    if locations is not None:
+        borg_call.append(locations)
+        print(borg_call)
     try:
-        os.chdir(os.path.expanduser(arguments.destination))
+        os.chdir(os.path.expanduser(destination))
-        subprocess.run(['borg', 'extract', REPOSITORY + '::' + arguments.name],
+        subprocess.run(borg_call, env=env, check=True)
-                       env=env, check=True)
     finally:
         os.chdir(prev_dir)
 
 
-def subcommand_export(arguments):
+def subcommand_export_tar(arguments):
     """Export archive contents as tarball."""
     # TODO: if this is only used for files in /tmp, add checks to verify that
-    # arguments.filename is within /tmp
+    # arguments.filename is within /tmp (does this actually increase security?)
     # TODO: arguments.filename is not a filename but a path
     path = os.path.dirname(arguments.filename)
     if not os.path.exists(path):
@@ -165,6 +189,32 @@ def subcommand_list_exports(arguments):
     print(json.dumps(exports))
 
 
+def _read_archive_file(archive, filepath):
+    """Read the content of a file inside an archive"""
+    arguments = ['borg', 'extract', archive, filepath, '--stdout']
+    return subprocess.check_output(arguments).decode()
+
+
+def subcommand_get_archive_apps(arguments):
+    """Get list of apps included in archive."""
+    manifest_folder = os.path.relpath(MANIFESTS_FOLDER, '/')
+    borg_call = ['borg', 'list', arguments.path, manifest_folder,
+            '--format', '{path}{NEWLINE}']
+    try:
+        manifest_path = subprocess.check_output(borg_call).decode().strip()
+    except subprocess.CalledProcessError:
+        sys.exit(1)
+
+    manifest = None
+    if manifest_path:
+        manifest_data = _read_archive_file(arguments.path,
+                manifest_path)
+        manifest = json.loads(manifest_data)
+    if manifest:
+        for app in manifest:
+            print(app['name'])
+
+
 def subcommand_get_export_apps(arguments):
     """Get list of apps included in exported archive file."""
     manifest = None
@@ -182,7 +232,17 @@ def subcommand_get_export_apps(arguments):
             print(app['name'])
 
 
-def subcommand_restore(arguments):
+def subcommand_restore_archive(arguments):
+    """Restore files from an archive."""
+    locations_data = ''.join(sys.stdin)
+    locations = json.loads(locations_data)
+
+    locations_string = " ".join(locations['directories'])
+    locations_string += " ".join(locations['files'])
+    _extract(arguments.path, arguments.destination, locations=locations_string)
+
+
+def subcommand_restore_exported_archive(arguments):
     """Restore files from an exported archive."""
     locations_data = ''.join(sys.stdin)
     locations = json.loads(locations_data)

plinth/modules/backups/__init__.py

@@ -30,7 +30,6 @@ from plinth.menu import main_menu
 from plinth.modules import storage
 
 from . import api
-from .manifest import backup
 
 version = 1
 
@@ -44,13 +43,14 @@ description = [
 
 service = None
 
-MANIFESTS_FOLDER = '/var/lib/plinth/backups-manifests/'
-
 BACKUP_FOLDER_NAME = 'FreedomBox-backups'
+DEFAULT_BACKUP_LOCATION = ('/var/lib/freedombox/', _('Root Filesystem'))
+MANIFESTS_FOLDER = '/var/lib/plinth/backups-manifests/'
+REPOSITORY = '/var/lib/freedombox/borgbackup'
+SESSION_BACKUP_VARIABLE = 'fbx-backup-filestamp'
 # default backup path for temporary actions like imports or download
 TMP_BACKUP_PATH = '/tmp/freedombox-backup.tar.gz'
 # session variable name that stores when a backup file should be deleted
-SESSION_BACKUP_VARIABLE = 'fbx-backup-filestamp'
 
 
 def init():
@@ -109,8 +109,10 @@ def create_archive(name, app_names):
 
 
 def delete_archive(name):
+    # TODO: is name actually a path?
     actions.superuser_run('backups', ['delete', '--name', name])
 
+
 def delete_tmp_backup_file():
     if os.path.isfile(TMP_BACKUP_PATH):
         os.remove(TMP_BACKUP_PATH)
@@ -121,21 +123,16 @@ def export_archive(name, location, tmp_dir=False):
     if tmp_dir:
         filepath = TMP_BACKUP_PATH
     else:
-        location_path = get_location_path(location)
+        filename = get_valid_filename(name) + '.tar.gz'
-        filepath = get_archive_path(location_path,
+        filepath = get_exported_archive_path(location, filename)
-                                    get_valid_filename(name) + '.tar.gz')
     # TODO: that's a full path, not a filename; rename argument
     actions.superuser_run('backups',
-                          ['export', '--name', name, '--filename', filepath])
+                          ['export-tar', '--name', name, '--filename', filepath])
 
 
 def get_export_locations():
     """Return a list of storage locations for exported backup archives."""
-    locations = [{
+    locations = [DEFAULT_BACKUP_LOCATION]
-        'path': '/var/lib/freedombox/',
-        'label': _('Root Filesystem'),
-        'device': '/'
-    }]
     if storage.is_running():
         devices = storage.udisks2.list_devices()
         for device in devices:
@@ -175,14 +172,27 @@ def get_export_files():
     return export_files
 
 
-def get_archive_path(location, archive_name):
+def get_archive_path(archive_name):
+    """Get path of an archive"""
+    return "::".join([REPOSITORY, archive_name])
+
+
+def get_exported_archive_path(location, archive_name):
+    """Get path of an exported archive"""
     return os.path.join(location, BACKUP_FOLDER_NAME, archive_name)
 
 
 def find_exported_archive(device, archive_name):
     """Return the full path for the exported archive file."""
     location_path = get_location_path(device)
-    return get_archive_path(location_path, archive_name)
+    return get_exported_archive_path(location_path, archive_name)
+
+
+def get_archive_apps(path):
+    """Get list of apps included in an archive."""
+    output = actions.superuser_run('backups',
+                                   ['get-archive-apps', '--path', path])
+    return output.splitlines()
 
 
 def get_export_apps(filename):
@@ -192,22 +202,36 @@ def get_export_apps(filename):
     return output.splitlines()
 
 
-def _restore_handler(packet):
+def _restore_exported_archive_handler(packet):
     """Perform restore operation on packet."""
     locations = {'directories': packet.directories, 'files': packet.files}
     locations_data = json.dumps(locations)
-    actions.superuser_run('backups', ['restore', '--filename', packet.label],
+    actions.superuser_run('backups', ['restore-exported-archive',
-                          input=locations_data.encode())
+        '--path', packet.label], input=locations_data.encode())
+
+
+def _restore_archive_handler(packet):
+    """Perform restore operation on packet."""
+    locations = {'directories': packet.directories, 'files': packet.files}
+    locations_data = json.dumps(locations)
+    actions.superuser_run('backups', ['restore-archive', '--path',
+                packet.label, '--destination', '/'], input=locations_data.encode())
 
 
 def restore_from_tmp(apps=None):
     """Restore files from temporary backup file"""
-    api.restore_apps(_restore_handler, app_names=apps, create_subvolume=False,
+    api.restore_apps(_restore_exported_archive_handler, app_names=apps,
-                 backup_file=TMP_BACKUP_PATH)
+                     create_subvolume=False, backup_file=TMP_BACKUP_PATH)
 
 
 def restore_exported(device, archive_name, apps=None):
     """Restore files from exported backup archive."""
     filename = find_exported_archive(device, archive_name)
-    api.restore_apps(_restore_handler, app_names=apps, create_subvolume=False,
+    api.restore_apps(_restore_exported_archive_handler, app_names=apps,
-                     backup_file=filename)
+                     create_subvolume=False, backup_file=filename)
+
+
+def restore(archive_path, apps=None):
+    """Restore files from a backup archive."""
+    api.restore_apps(_restore_archive_handler, app_names=apps,
+                     create_subvolume=False, backup_file=archive_path)

plinth/modules/backups/api.py

@@ -115,6 +115,7 @@ class Packet:
         self.scope = scope
         self.root = root
         self.apps = apps
+        # TODO: label is an archive path -- rename
         self.label = label
         self.errors = []
 

plinth/modules/backups/forms.py

@@ -26,7 +26,8 @@ from django.core.validators import FileExtensionValidator
 from django.utils.translation import ugettext, ugettext_lazy as _
 
 from . import api
-from . import get_export_locations, get_archive_path, get_location_path
+from . import get_export_locations, get_exported_archive_path, \
+        get_location_path
 
 
 def _get_app_choices(apps):
@@ -84,9 +85,8 @@ class RestoreFromTmpForm(forms.Form):
         """Initialize the form with selectable apps."""
         apps = kwargs.pop('apps')
         super().__init__(*args, **kwargs)
-        self.fields['selected_apps'].choices = [
+        self.fields['selected_apps'].choices = _get_app_choices(apps)
-            (app[0], app[1].name) for app in apps]
+        self.fields['selected_apps'].initial = [app.name for app in apps]
-        self.fields['selected_apps'].initial = [app[0] for app in apps]
 
 
 class RestoreForm(forms.Form):
@@ -131,7 +131,7 @@ class UploadForm(forms.Form):
         location_path = get_location_path(location_device)
         # if other errors occured before, 'file' won't be in cleaned_data
         if (file and file.name):
-            filepath = get_archive_path(location_path, file.name)
+            filepath = get_exported_archive_path(location_path, file.name)
             if os.path.exists(filepath):
                 raise forms.ValidationError(
                     "File %s already exists" % file.name)

plinth/modules/backups/templates/backups.html

@@ -82,14 +82,20 @@
           <tr id="archive-{{ archive.name }}" class="archive">
             <td class="archive-name">{{ archive.name }}</td>
             <td class="archive-operations">
+              <!-- TODO: use or remove
               <a class="archive-export btn btn-sm btn-default"
                  href="{% url 'backups:export' archive.name %}">
                 {% trans "Export" %}
               </a>
+              -->
               <a class="archive-export btn btn-sm btn-default" target="_blank"
                  href="{% url 'backups:export-and-download' archive.name %}">
                 {% trans "Download" %}
               </a>
+              <a class="archive-export btn btn-sm btn-default"
+                 href="{% url 'backups:restore-archive' archive.name %}">
+                {% trans "Restore" %}
+              </a>
               <a class="archive-delete btn btn-sm btn-default"
                  href="{% url 'backups:delete' archive.name %}">
                 <span class="glyphicon glyphicon-trash" aria-hidden="true">

plinth/modules/backups/urls.py

@@ -22,7 +22,7 @@ from django.conf.urls import url
 
 from .views import IndexView, CreateArchiveView, DownloadArchiveView, \
     DeleteArchiveView, ExportArchiveView, RestoreView, UploadArchiveView, \
-    ExportAndDownloadView, RestoreFromTmpView
+    ExportAndDownloadView, RestoreArchiveView, RestoreFromTmpView
 
 urlpatterns = [
     url(r'^sys/backups/$', IndexView.as_view(), name='index'),
@@ -38,6 +38,8 @@ urlpatterns = [
     url(r'^sys/backups/upload/$', UploadArchiveView.as_view(), name='upload'),
     url(r'^sys/backups/restore/(?P<device>[^/]+)/(?P<name>[^/]+)/$',
         RestoreView.as_view(), name='restore'),
+    url(r'^sys/backups/restore-archive/(?P<name>[^/]+)/$',
+        RestoreArchiveView.as_view(), name='restore-archive'),
     url(r'^sys/backups/restore-from-tmp/$',
         RestoreFromTmpView.as_view(), name='restore-from-tmp'),
 ]

plinth/modules/backups/views.py

@@ -140,21 +140,15 @@ class ExportAndDownloadView(View):
     """View to export and download an archive."""
     def get(self, request, name):
         name = unquote(name)
-        with create_temporary_backup_file(name) as filename, \
+        filename = "%s.tar.gz" % name
-                open(filename, 'rb') as file_handle:
+        with create_temporary_backup_file(name) as filepath:
-            (content_type, encoding) = mimetypes.guess_type(filename)
+            return _get_file_response(filepath, filename)
-            response = HttpResponse(File(file_handle),
-                content_type=content_type)
-            content_disposition = 'attachment; filename="%s"' % filename
-            response['Content-Disposition'] = content_disposition
-            if encoding:
-                response['Content-Encoding'] = encoding
-
-            return response
 
 
 class create_temporary_backup_file:
     """Create a temporary backup file that gets deleted after using it"""
+    # TODO: try using export-tar with FILE parameter '-' and reading stdout:
+    # https://borgbackup.readthedocs.io/en/stable/usage/tar.html
 
     def __init__(self, name):
         self.name = name
@@ -287,7 +281,7 @@ class RestoreFromTmpView(SuccessMessageMixin, FormView):
         included_apps = self._get_included_apps()
         installed_apps = api.get_all_apps_for_backup()
         kwargs['apps'] = [
-            app for app in installed_apps if app[0] in included_apps
+            app for app in installed_apps if app.name in included_apps
         ]
         return kwargs
 
@@ -301,3 +295,41 @@ class RestoreFromTmpView(SuccessMessageMixin, FormView):
         """Restore files from the archive on valid form submission."""
         backups.restore_from_tmp(form.cleaned_data['selected_apps'])
         return super().form_valid(form)
+
+
+class RestoreArchiveView(SuccessMessageMixin, FormView):
+    """View to restore files from an archive."""
+    form_class = forms.RestoreForm
+    prefix = 'backups'
+    template_name = 'backups_restore.html'
+    success_url = reverse_lazy('backups:index')
+    success_message = _('Restored files from backup.')
+
+    def _get_included_apps(self):
+        """Save some data used to instantiate the form."""
+        name = unquote(self.kwargs['name'])
+        archive_path = backups.get_archive_path(name)
+        return backups.get_archive_apps(archive_path)
+
+    def get_form_kwargs(self):
+        """Pass additional keyword args for instantiating the form."""
+        kwargs = super().get_form_kwargs()
+        included_apps = self._get_included_apps()
+        installed_apps = api.get_all_apps_for_backup()
+        kwargs['apps'] = [
+            app for app in installed_apps if app.name in included_apps
+        ]
+        return kwargs
+
+    def get_context_data(self, **kwargs):
+        """Return additional context for rendering the template."""
+        context = super().get_context_data(**kwargs)
+        context['title'] = _('Restore from backup')
+        context['name'] = self.kwargs['name']
+        return context
+
+    def form_valid(self, form):
+        """Restore files from the archive on valid form submission."""
+        archive_path = backups.get_archive_path(self.kwargs['name'])
+        backups.restore(archive_path, form.cleaned_data['selected_apps'])
+        return super().form_valid(form)