setup: Fix Plinth startup issues

- Selectively run setup for modules
- Middleware pending

Fixes #1024

Signed-off-by: Joseph Nuthalpati <njoseph@thoughtworks.com>
Signed-off-by: Sunil Mohan Adapa <sunil@medhas.org>
Reviewed-by: Sunil Mohan Adapa <sunil@medhas.org>

plinth/__main__.py

@@ -155,6 +155,11 @@ def setup_server():
     cherrypy.engine.signal_handler.subscribe()
 
 
+def on_server_stop():
+    """Stop all other threads since web server is trying to exit"""
+    setup.stop()
+
+
 def configure_django():
     """Setup Django configuration in the absense of .settings file"""
     logging_configuration = {
@@ -218,7 +223,7 @@ def configure_django():
 
     django.conf.settings.configure(
         ALLOWED_HOSTS=['*'],
-        AUTH_PASSWORD_VALIDATORS = [
+        AUTH_PASSWORD_VALIDATORS=[
             {
                 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
             },
@@ -259,6 +264,7 @@ def configure_django():
             'plinth.middleware.AdminRequiredMiddleware',
             'plinth.modules.first_boot.middleware.FirstBootMiddleware',
             'plinth.middleware.SetupMiddleware',
+            'plinth.middleware.FirstSetupMiddleware',
         ),
         ROOT_URLCONF='plinth.urls',
         SECURE_PROXY_SSL_HEADER=secure_proxy_ssl_header,
@@ -279,21 +285,14 @@ def configure_django():
     os.chmod(cfg.store_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP)
 
 
-def run_setup(module_list, allow_install=True):
-    try:
-        if not module_list:
-            setup.setup_modules(essential=True, allow_install=allow_install)
-        else:
-            setup.setup_modules(module_list, allow_install=allow_install)
-    except Exception as exception:
-        logger.error('Error running setup - %s', exception)
-        return 1
-    return 0
-
-
 def run_setup_and_exit(module_list, allow_install=True):
     """Run setup on all essential modules and exit."""
-    error_code = run_setup(module_list, allow_install)
+    error_code = 0
+    try:
+        setup.run_setup_on_modules(module_list, allow_install)
+    except Exception as exception:
+        error_code = 1
+
     sys.exit(error_code)
 
 
@@ -380,13 +379,11 @@ def main():
     if arguments.diagnose:
         run_diagnostics_and_exit()
 
-    # Run setup steps for essential modules
+    setup.run_setup_in_background()
-    # Installation is not necessary as they are dependencies of Plinth
-    run_setup(None, allow_install=False)
-
     setup_server()
 
     cherrypy.engine.start()
+    cherrypy.engine.subscribe('stop', on_server_stop)
     cherrypy.engine.block()
 
 

plinth/middleware.py

@@ -14,7 +14,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
-
 """
 Common Django middleware.
 """
@@ -24,17 +23,18 @@ from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.decorators import login_required
 from django.core.exceptions import PermissionDenied
+from django.shortcuts import render
 from django.utils.translation import ugettext_lazy as _
 import logging
 
 from stronghold.utils import is_view_func_public
 
 import plinth
+from plinth import setup
 from plinth.package import PackageException
 from plinth.utils import is_user_admin
 from . import views
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -76,7 +76,7 @@ class SetupMiddleware(object):
                     error_details = getattr(exception, 'error_details', '')
                     message = _('Error installing application: {string} '
                                 '{details}').format(
-                        string=error_string, details=error_details)
+                                    string=error_string, details=error_details)
                 else:
                     message = _('Error installing application: {error}') \
                         .format(error=exception)
@@ -104,3 +104,15 @@ class AdminRequiredMiddleware(object):
 
         if not is_user_admin(request):
             raise PermissionDenied
+
+
+class FirstSetupMiddleware(object):
+
+    @staticmethod
+    def process_view(request, view_func, view_args, view_kwargs):
+        """Block all user interactions when first setup is pending."""
+        if not setup.is_first_setup_running:
+            return
+
+        context = {'is_first_setup_running': setup.is_first_setup_running}
+        return render(request, 'first_setup.html', context)

plinth/package.py

@@ -131,3 +131,13 @@ class Transaction(object):
         }
         self.status_string = status_map.get(parts[0], '')
         self.percentage = int(float(parts[2]))
+
+
+def is_package_manager_busy():
+    """Return whether a package manager is running."""
+    try:
+        actions.superuser_run('packages', ['is-package-manager-busy'])
+        return True
+    except actions.ActionError:
+        return False
+

plinth/setup.py

@@ -14,7 +14,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
-
 """
 Plinth module with utilites for performing application setup operations.
 """
@@ -22,13 +21,20 @@ Plinth module with utilites for performing application setup operations.
 import apt
 import logging
 import threading
+import time
 
 from . import package
 from .errors import PackageNotInstalledError
 import plinth
+from plinth import actions
+from plinth import package
 
 logger = logging.getLogger(__name__)
 
+_is_first_setup = False
+is_first_setup_running = False
+_is_shutting_down = False
+
 
 class Helper(object):
     """Helper routines for modules to show progress."""
@@ -163,12 +169,18 @@ def init(module_name, module):
         module.setup_helper = Helper(module_name, module)
 
 
+def stop():
+    """Set a flag to indicate that the setup process must stop."""
+    global _is_shutting_down
+    _is_shutting_down = True
+
+
 def setup_modules(module_list=None, essential=False, allow_install=True):
     """Run setup on selected or essential modules."""
     logger.info('Running setup for modules, essential - %s, '
                 'selected modules - %s', essential, module_list)
     for module_name, module in plinth.module_loader.loaded_modules.items():
-        if essential and not getattr(module, 'is_essential', False):
+        if essential and not is_module_essential(module):
             continue
 
         if module_list and module_name not in module_list:
@@ -180,7 +192,7 @@ def setup_modules(module_list=None, essential=False, allow_install=True):
 def list_dependencies(module_list=None, essential=False):
     """Print list of packages required by selected or essential modules."""
     for module_name, module in plinth.module_loader.loaded_modules.items():
-        if essential and not getattr(module, 'is_essential', False):
+        if essential and not is_module_essential(module):
             continue
 
         if module_list and module_name not in module_list and \
@@ -189,3 +201,113 @@ def list_dependencies(module_list=None, essential=False):
 
         for package_name in getattr(module, 'managed_packages', []):
             print(package_name)
+
+
+def run_setup_in_background():
+    """Run setup in a background thread."""
+    global _is_first_setup
+    _set_is_first_setup()
+    threading.Thread(target=_run_setup).start()
+
+
+def _run_setup():
+    """Run setup with retry till it succeeds."""
+    sleep_time = 10
+    while True:
+        try:
+            if _is_first_setup:
+                logger.info('Running first setup.')
+                _run_first_setup()
+                break
+            else:
+                logger.info('Running regular setup.')
+                _run_regular_setup()
+                break
+        except Exception as ex:
+            logger.warning('Unable to complete setup: %s', ex)
+            logger.info('Will try again in {} seconds'.format(sleep_time))
+            time.sleep(sleep_time)
+            if _is_shutting_down:
+                break
+
+    logger.info('Setup thread finished.')
+
+
+def _run_first_setup():
+    """Run setup on essential modules on first setup."""
+    global is_first_setup_running
+    is_first_setup_running = True
+    # TODO When it errors out, show error in the UI
+    run_setup_on_modules(None, allow_install=False)
+    is_first_setup_running = False
+
+
+def _run_regular_setup():
+    """Run setup on all modules also installing required packages."""
+    # TODO show notification that upgrades are running
+    if package.is_package_manager_busy():
+        raise Exception('Package manager is busy.')
+
+    all_modules = _get_modules_for_regular_setup()
+    run_setup_on_modules(all_modules, allow_install=True)
+
+
+def _get_modules_for_regular_setup():
+    all_modules = plinth.module_loader.loaded_modules.items()
+
+    def is_setup_required(module):
+        """Setup is required for:
+        1. essential modules that are not up-to-date
+        2. non-essential modules that are installed and need updates
+        """
+        if (is_module_essential(module) and
+            not module_state_matches(module, 'up-to-date')):
+            return True
+
+        if module_state_matches(module, 'needs-update'):
+            return True
+
+        return False
+
+    return [name
+            for name, module in all_modules
+            if is_setup_required(module)]
+
+
+def is_module_essential(module):
+    return getattr(module, 'is_essential', False)
+
+
+def module_state_matches(module, state):
+    return module.setup_helper.get_state() == state
+
+
+def _set_is_first_setup():
+    """Return whether all essential modules have been setup at least once."""
+    global _is_first_setup
+    modules = plinth.module_loader.loaded_modules.values()
+    _is_first_setup = any(
+        (module
+         for module in modules
+         if is_module_essential(module) and module_state_matches(module, 'needs-setup')))
+
+
+def run_setup_on_modules(module_list, allow_install=True):
+    """Run setup on the given list of modules.
+
+    module_list is the list of modules to run setup on. If None is given, run
+    setup on all essential modules only.
+
+    allow_install with or without package installation. When setting up
+    essential modules, installing packages is not required as Plinth itself has
+    dependencies on all essential modules.
+
+    """
+    try:
+        if not module_list:
+            setup_modules(essential=True, allow_install=allow_install)
+        else:
+            setup_modules(module_list, allow_install=allow_install)
+    except Exception as exception:
+        logger.error('Error running setup - %s', exception)
+        raise

plinth/templates/first_setup.html

@@ -0,0 +1,45 @@
+{% extends "base.html" %}
+{% comment %}
+#
+# This file is part of Plinth.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+{% endcomment %}
+
+{% load bootstrap %}
+{% load i18n %}
+
+{% block page_head %}
+
+  {% if setup_helper.current_operation %}
+    <meta http-equiv="refresh" content="3" />
+  {% endif %}
+
+{% endblock %}
+
+
+{% block content %}
+
+  {% if is_first_setup_running %}
+    <div class="alert alert-warning" role="alert">
+      {% blocktrans trimmed %}
+        Please wait for {{ box_name }} to finish installation.
+        You can start using your {{ box_name }} once it is done.
+      {% endblocktrans %}
+    </div>
+  {% endif %}
+
+{% endblock %}
+

plinth/views.py

@@ -31,6 +31,7 @@ import time
 from . import forms, frontpage
 import plinth
 from plinth import actions
+from plinth import package
 from plinth.modules.storage import views as disk_views
 
 
@@ -140,17 +141,9 @@ class SetupView(TemplateView):
         """Return the context data rendering the template."""
         context = super(SetupView, self).get_context_data(**kwargs)
         context['setup_helper'] = self.kwargs['setup_helper']
-        context['package_manager_is_busy'] = self.is_package_manager_busy()
+        context['package_manager_is_busy'] = package.is_package_manager_busy()
         return context
 
-    def is_package_manager_busy(self):
-        """Return whether a package manager is running."""
-        try:
-            actions.superuser_run('packages', ['is-package-manager-busy'])
-            return True
-        except actions.ActionError:
-            return False
-
     def post(self, *args, **kwargs):
         """Handle installing/upgrading applications.