upgrades: Make manual upgrade a background process

- Closes #366 and closes #304 (all sub-tasks).

- Start new process group with setsid() by sending
  start_new_session=True

- Detach from parent process fds by closing all FDs and attaching stdin,
  stdou and stderr to /dev/null.

- Don't wait for the process to complete.

- This allows for upgrading Plinth while upgrades are trigged from
  Plinth itself.

- Show log of upgrade exection instead of output and error log of the
  process which can no longer be collected.  This has the advantage of
  showing automatic executions also.

- Rewrite the mechanism to detect whether upgrades can be run.  It is
  now based on whether the package manager is busy.  This has the
  advantage of working properly if other apt processes are running,
  automatic upgrades are running, etc.

- Busy status works even if Plinth is restarted while upgrades are in
  progress.

- More descriptive messages showing that upgrades don't have to be
  triggered manually.

- Warn that other packages can't be installed while upgrades are
  running, which may take a long time.

- Warn the users of potential temporary unavailability of
  Plinth/Apache2.

actions/upgrades

@@ -61,7 +61,11 @@ def subcommand_run(_):
         sys.exit(1)
 
     try:
-        subprocess.check_call(['unattended-upgrades', '-v'])
+        subprocess.Popen(
+            ['unattended-upgrades', '-v'],
+            stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL, close_fds=True,
+            start_new_session=True)
     except FileNotFoundError:
         print('Error: unattended-upgrades is not available.', file=sys.stderr)
         sys.exit(2)

plinth/modules/upgrades/templates/upgrades.html

@@ -22,7 +22,7 @@
 
 {% block page_head %}
 
-  {% if running %}
+  {% if is_busy %}
     <meta http-equiv="refresh" content="3"/>
   {% endif %}
 
@@ -33,47 +33,25 @@
 
   <h2>{{ title }}</h2>
 
-  {% if result %}
+  <p>
+    {% blocktrans trimmed %}
+      Upgrades install the latest software and security updates. When automatic
+      upgrades are enabled, upgrades are automatically run every night. You
+      don't normally need to start the upgrade process.
+    {% endblocktrans %}
+  </p>
 
-    {% if result.return_code %}
-      <p>
-        {% trans "There was an error while upgrading." %}
-      </p>
+  <p>
+    {% blocktrans trimmed %}
+      Depending on the number of packages to install, this may take a long time
+      to complete.  While upgrades are in progress, you will not be able to
+      install other packages.  During the upgrade, this web interface may be
+      temporarily unavailable and show an error.  Refresh the page to continue.
+    {% endblocktrans %}
+  </p>
 
-      <h5>{% trans "Output from unattended-upgrades:" %}</h5>
-      <pre>{{ result.error }}</pre>
-      {% if result.output %}
-        <pre>{{ result.output }}</pre>
-      {% endif %}
-    {% else %}
-      <p>
-        {% trans "The operating system is up to date now. &nbsp;" %}
-        <button type="button" class="btn btn-default show-details"
-                style='display: none;'>
-          {% trans "Show Details" %}
-          <div class="caret"></div>
-        </button>
-      </p>
-
-      <div class="details">
-        <h5>{% trans "Output from unattended-upgrades:" %}</h5>
-        <pre>{{ result.output }}</pre>
-      </div>
-    {% endif %}
-
-  {% endif %}
-
-
-  {% if not result and not running %}
-    <p>
-      {% blocktrans trimmed %}
-        This will run unattended-upgrades, which will attempt to upgrade
-        your system with the latest Debian packages. It may take a few
-        minutes to complete.
-      {% endblocktrans %}
-    </p>
-
-    <form class="form" method="post" action="{% url 'upgrades:run' %}">
+  {% if not is_busy %}
+    <form class="form" method="post" action="{% url 'upgrades:upgrade' %}">
       {% csrf_token %}
 
       <input type="submit" class="btn btn-primary"
@@ -81,13 +59,19 @@
     </form>
   {% endif %}
 
-  {% if running %}
+  {% if is_busy %}
     <p class="running-status-parent">
       <span class="running-status active"></span>
-      {% trans "System is being upgraded." %}
+      {% trans "A package manager is running." %}
     </p>
   {% endif %}
 
+  {% if log %}
+    <h5>{% trans "Recent log from upgrades:" %}</h5>
+
+    <pre>{{ log }}</pre>
+  {% endif %}
+
 {% endblock %}
 
 {% block page_js %}

plinth/modules/upgrades/urls.py

@@ -27,5 +27,4 @@ from . import views
 urlpatterns = [
     url(r'^sys/upgrades/$', views.index, name='index'),
     url(r'^sys/upgrades/upgrade/$', views.upgrade, name='upgrade'),
-    url(r'^sys/upgrades/upgrade/run/$', views.run, name='run'),
 ]

plinth/modules/upgrades/views.py

@@ -25,6 +25,7 @@ from django.shortcuts import redirect
 from django.template.response import TemplateResponse
 from django.utils.translation import ugettext as _, ugettext_lazy
 from django.views.decorators.http import require_POST
+import subprocess
 
 from .forms import ConfigureForm
 from plinth import actions
@@ -36,7 +37,8 @@ subsubmenu = [{'url': reverse_lazy('upgrades:index'),
               {'url': reverse_lazy('upgrades:upgrade'),
                'text': ugettext_lazy('Upgrade Packages')}]
 
-upgrade_process = None
+LOG_FILE = '/var/log/unattended-upgrades/unattended-upgrades.log'
+LOCK_FILE = '/var/log/dpkg/lock'
 
 
 def on_install():
@@ -65,29 +67,42 @@ def index(request):
                              'form': form,
                              'subsubmenu': subsubmenu})
 
+def is_package_manager_busy():
+    """Return whether a package manager is running."""
+    try:
+        subprocess.check_output(['lsof', '/var/lib/dpkg/lock'])
+        return True
+    except subprocess.CalledProcessError:
+        return False
+
+
+def get_log():
+    """Return the current log for unattended upgrades."""
+    try:
+        with open(LOG_FILE, 'r') as file_handle:
+            return file_handle.read()
+    except IOError:
+        return None
+
 
 @package.required(['unattended-upgrades'], on_install=on_install)
 def upgrade(request):
     """Serve the upgrade page."""
-    result = _collect_upgrade_result(request)
+    is_busy = is_package_manager_busy()
+
+    if request.method == 'POST':
+        try:
+            actions.superuser_run('upgrades', ['run'])
+            messages.success(request, _('Upgrade process started.'))
+            is_busy = True
+        except ActionError:
+            messages.error(request, _('Starting upgrade failed.'))
 
     return TemplateResponse(request, 'upgrades.html',
                             {'title': _('Package Upgrades'),
                              'subsubmenu': subsubmenu,
-                             'running': bool(upgrade_process),
-                             'result': result})
-
-
-@require_POST
-@package.required(['unattended-upgrades'], on_install=on_install)
-def run(_):
-    """Start the upgrade process."""
-    global upgrade_process
-    if not upgrade_process:
-        upgrade_process = actions.superuser_run(
-            'upgrades', ['run'], async=True)
-
-    return redirect('upgrades:upgrade')
+                             'is_busy': is_busy,
+                             'log': get_log()})
 
 
 def get_status():
@@ -121,30 +136,3 @@ def _apply_changes(request, old_status, new_status):
         messages.success(request, _('Automatic upgrades enabled'))
     else:
         messages.success(request, _('Automatic upgrades disabled'))
-
-
-def _collect_upgrade_result(request):
-    """Handle upgrade process completion."""
-    global upgrade_process
-    if not upgrade_process:
-        return
-
-    return_code = upgrade_process.poll()
-
-    # Upgrade process is not complete yet
-    if return_code is None:
-        return
-
-    output, error = upgrade_process.communicate()
-    output, error = output.decode(), error.decode()
-
-    if not return_code:
-        messages.success(request, _('Upgrade completed.'))
-    else:
-        messages.error(request, _('Upgrade failed.'))
-
-    upgrade_process = None
-
-    return {'return_code': return_code,
-            'output': output,
-            'error': error}