mirror of
https://github.com/freedombox/FreedomBox.git
synced 2026-01-21 07:55:00 +00:00
f59fc5e33b
- Also add ability restore missing configuration files during reinstall. - Reinstall is useful for restoring the original configuration files of the package. Signed-off-by: Sunil Mohan Adapa <sunil@medhas.org> Reviewed-by: James Valleroy <jvalleroy@mailbox.org>
586 lines
20 KiB
Python
586 lines
20 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
"""
|
|
Utilities for performing application setup operations.
|
|
"""
|
|
|
|
import importlib
|
|
import logging
|
|
import os
|
|
import threading
|
|
import time
|
|
from collections import defaultdict
|
|
|
|
import apt
|
|
|
|
import plinth
|
|
from plinth.signals import post_setup
|
|
|
|
from . import package
|
|
from .errors import PackageNotInstalledError
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_is_first_setup = False
|
|
is_first_setup_running = False
|
|
_is_shutting_down = False
|
|
|
|
_force_upgrader = None
|
|
|
|
|
|
class Helper(object):
|
|
"""Helper routines for modules to show progress."""
|
|
|
|
def __init__(self, module_name, module):
|
|
"""Initialize the object."""
|
|
self.module_name = module_name
|
|
self.module = module
|
|
self.current_operation = None
|
|
self.is_finished = None
|
|
self.exception = None
|
|
self.allow_install = True
|
|
|
|
def run_in_thread(self):
|
|
"""Execute the setup process in a thread."""
|
|
thread = threading.Thread(target=self._run)
|
|
thread.start()
|
|
|
|
def _run(self):
|
|
"""Collect exceptions when running in a thread."""
|
|
try:
|
|
self.run()
|
|
except Exception as exception:
|
|
self.exception = exception
|
|
|
|
def collect_result(self):
|
|
"""Return the exception if any."""
|
|
exception = self.exception
|
|
self.exception = None
|
|
self.is_finished = None
|
|
return exception
|
|
|
|
def run(self, allow_install=True):
|
|
"""Execute the setup process."""
|
|
# Setup for the module is already running
|
|
if self.current_operation:
|
|
return
|
|
|
|
current_version = self.get_setup_version()
|
|
if current_version >= self.module.version:
|
|
return
|
|
|
|
self.allow_install = allow_install
|
|
self.exception = None
|
|
self.current_operation = None
|
|
self.is_finished = False
|
|
try:
|
|
if hasattr(self.module, 'setup'):
|
|
logger.info('Running module setup - %s', self.module_name)
|
|
self.module.setup(self, old_version=current_version)
|
|
else:
|
|
logger.info('Module does not require setup - %s',
|
|
self.module_name)
|
|
except Exception as exception:
|
|
logger.exception('Error running setup - %s', exception)
|
|
raise exception
|
|
else:
|
|
self.set_setup_version(self.module.version)
|
|
post_setup.send_robust(sender=self.__class__,
|
|
module_name=self.module_name)
|
|
finally:
|
|
self.is_finished = True
|
|
self.current_operation = None
|
|
|
|
def install(self, package_names, skip_recommends=False,
|
|
force_configuration=None, reinstall=False,
|
|
force_missing_configuration=False):
|
|
"""Install a set of packages marking progress."""
|
|
if self.allow_install is False:
|
|
# Raise error if packages are not already installed.
|
|
cache = apt.Cache()
|
|
for package_name in package_names:
|
|
if not cache[package_name].is_installed:
|
|
raise PackageNotInstalledError(package_name)
|
|
|
|
return
|
|
|
|
logger.info('Running install for module - %s, packages - %s',
|
|
self.module_name, package_names)
|
|
|
|
transaction = package.Transaction(self.module_name, package_names)
|
|
self.current_operation = {
|
|
'step': 'install',
|
|
'transaction': transaction,
|
|
}
|
|
transaction.install(skip_recommends, force_configuration, reinstall,
|
|
force_missing_configuration)
|
|
|
|
def call(self, step, method, *args, **kwargs):
|
|
"""Call an arbitrary method during setup and note down its stage."""
|
|
logger.info('Running step for module - %s, step - %s',
|
|
self.module_name, step)
|
|
self.current_operation = {'step': step}
|
|
return method(*args, **kwargs)
|
|
|
|
def get_state(self):
|
|
"""Return whether the module is not setup or needs upgrade."""
|
|
current_version = self.get_setup_version()
|
|
if current_version and self.module.version <= current_version:
|
|
return 'up-to-date'
|
|
|
|
# If a module need installing/updating but no setup method is
|
|
# available, then automatically set version.
|
|
#
|
|
# Minor violation of 'get' only discipline for convenience.
|
|
if not hasattr(self.module, 'setup'):
|
|
self.set_setup_version(self.module.version)
|
|
return 'up-to-date'
|
|
|
|
if not current_version:
|
|
return 'needs-setup'
|
|
|
|
return 'needs-update'
|
|
|
|
def get_setup_version(self):
|
|
"""Return the setup version of a module."""
|
|
# XXX: Optimize version gets
|
|
from . import models
|
|
|
|
try:
|
|
module_entry = models.Module.objects.get(pk=self.module_name)
|
|
return module_entry.setup_version
|
|
except models.Module.DoesNotExist:
|
|
return 0
|
|
|
|
def set_setup_version(self, version):
|
|
"""Set a module's setup version."""
|
|
from . import models
|
|
|
|
models.Module.objects.update_or_create(
|
|
pk=self.module_name, defaults={'setup_version': version})
|
|
|
|
def has_unavailable_packages(self):
|
|
"""Find if any of the packages managed by the module are not available.
|
|
Returns True if one or more of the packages is not available in the
|
|
user's Debian distribution or False otherwise.
|
|
Returns None if it cannot be reliably determined whether the
|
|
packages are available or not.
|
|
"""
|
|
APT_LISTS_DIR = '/var/lib/apt/lists/'
|
|
num_files = len([
|
|
name for name in os.listdir(APT_LISTS_DIR)
|
|
if os.path.isfile(os.path.join(APT_LISTS_DIR, name))
|
|
])
|
|
if num_files < 2: # not counting the lock file
|
|
return None
|
|
cache = apt.Cache()
|
|
managed_pkgs = _get_module_managed_packages(self.module)
|
|
unavailable_pkgs = (pkg_name for pkg_name in managed_pkgs
|
|
if pkg_name not in cache)
|
|
return any(unavailable_pkgs)
|
|
|
|
|
|
def init(module_name, module):
|
|
"""Create a setup helper for a module for later use."""
|
|
if not hasattr(module, 'setup_helper'):
|
|
module.setup_helper = Helper(module_name, module)
|
|
|
|
|
|
def stop():
|
|
"""Set a flag to indicate that the setup process must stop."""
|
|
global _is_shutting_down
|
|
_is_shutting_down = True
|
|
|
|
if _force_upgrader:
|
|
_force_upgrader.shutdown()
|
|
|
|
|
|
def setup_modules(module_list=None, essential=False, allow_install=True):
|
|
"""Run setup on selected or essential modules."""
|
|
logger.info(
|
|
'Running setup for modules, essential - %s, '
|
|
'selected modules - %s', essential, module_list)
|
|
for module_name, module in plinth.module_loader.loaded_modules.items():
|
|
if essential and not _is_module_essential(module):
|
|
continue
|
|
|
|
if module_list and module_name not in module_list:
|
|
continue
|
|
|
|
module.setup_helper.run(allow_install=allow_install)
|
|
|
|
|
|
def list_dependencies(module_list=None, essential=False):
|
|
"""Print list of packages required by selected or essential modules."""
|
|
for module_import_path in plinth.module_loader.get_modules_to_load():
|
|
module_name = module_import_path.split('.')[-1]
|
|
module = importlib.import_module(module_import_path)
|
|
if essential and not _is_module_essential(module):
|
|
continue
|
|
|
|
if module_list and module_name not in module_list and \
|
|
'*' not in module_list:
|
|
continue
|
|
|
|
for package_name in _get_module_managed_packages(module):
|
|
print(package_name)
|
|
|
|
|
|
def run_setup_in_background():
|
|
"""Run setup in a background thread."""
|
|
_set_is_first_setup()
|
|
threading.Thread(target=_run_setup).start()
|
|
|
|
|
|
def _run_setup():
|
|
"""Run setup with retry till it succeeds."""
|
|
sleep_time = 10
|
|
while True:
|
|
try:
|
|
if _is_first_setup:
|
|
logger.info('Running first setup.')
|
|
_run_first_setup()
|
|
break
|
|
else:
|
|
logger.info('Running regular setup.')
|
|
_run_regular_setup()
|
|
break
|
|
except Exception as ex:
|
|
logger.warning('Unable to complete setup: %s', ex)
|
|
logger.info('Will try again in {} seconds'.format(sleep_time))
|
|
time.sleep(sleep_time)
|
|
if _is_shutting_down:
|
|
break
|
|
|
|
logger.info('Setup thread finished.')
|
|
|
|
|
|
def _run_first_setup():
|
|
"""Run setup on essential modules on first setup."""
|
|
global is_first_setup_running
|
|
is_first_setup_running = True
|
|
# TODO When it errors out, show error in the UI
|
|
run_setup_on_modules(None, allow_install=False)
|
|
is_first_setup_running = False
|
|
|
|
|
|
def _run_regular_setup():
|
|
"""Run setup on all modules also installing required packages."""
|
|
# TODO show notification that upgrades are running
|
|
if package.is_package_manager_busy():
|
|
raise Exception('Package manager is busy.')
|
|
|
|
all_modules = _get_modules_for_regular_setup()
|
|
run_setup_on_modules(all_modules, allow_install=True)
|
|
|
|
|
|
def _get_modules_for_regular_setup():
|
|
all_modules = plinth.module_loader.loaded_modules.items()
|
|
|
|
def is_setup_required(module):
|
|
"""Setup is required for:
|
|
1. essential modules that are not up-to-date
|
|
2. non-essential modules that are installed and need updates
|
|
"""
|
|
if _is_module_essential(module) and \
|
|
not _module_state_matches(module, 'up-to-date'):
|
|
return True
|
|
|
|
if _module_state_matches(module, 'needs-update'):
|
|
return True
|
|
|
|
return False
|
|
|
|
return [name for name, module in all_modules if is_setup_required(module)]
|
|
|
|
|
|
def _is_module_essential(module):
|
|
"""Return if a module is an essential module."""
|
|
return getattr(module, 'is_essential', False)
|
|
|
|
|
|
def _get_module_managed_packages(module):
|
|
"""Return list of packages managed by a module."""
|
|
return getattr(module, 'managed_packages', [])
|
|
|
|
|
|
def _module_state_matches(module, state):
|
|
"""Return if the current setup state of a module matches given state."""
|
|
return module.setup_helper.get_state() == state
|
|
|
|
|
|
def _set_is_first_setup():
|
|
"""Set whether all essential modules have been setup at least once."""
|
|
global _is_first_setup
|
|
modules = plinth.module_loader.loaded_modules.values()
|
|
_is_first_setup = any((module for module in modules
|
|
if _is_module_essential(module)
|
|
and _module_state_matches(module, 'needs-setup')))
|
|
|
|
|
|
def run_setup_on_modules(module_list, allow_install=True):
|
|
"""Run setup on the given list of modules.
|
|
|
|
module_list is the list of modules to run setup on. If None is given, run
|
|
setup on all essential modules only.
|
|
|
|
allow_install with or without package installation. When setting up
|
|
essential modules, installing packages is not required as FreedomBox
|
|
(Plinth) itself has dependencies on all essential modules.
|
|
|
|
"""
|
|
try:
|
|
if not module_list:
|
|
setup_modules(essential=True, allow_install=allow_install)
|
|
else:
|
|
setup_modules(module_list, allow_install=allow_install)
|
|
except Exception as exception:
|
|
logger.error('Error running setup - %s', exception)
|
|
raise
|
|
|
|
|
|
class ForceUpgrader():
|
|
"""Find and upgrade packages by force when conffile prompt is needed.
|
|
|
|
Primary duty here is take responsibility of packages that will be rejected
|
|
by unattended-upgrades for upgrades. Only packages that apps manage are
|
|
handled as that is only case in which we know how to deal with
|
|
configuration file changes. Roughly, we will read old configuration file,
|
|
force upgrade the package with new configuration and then apply the old
|
|
preferences on it.
|
|
|
|
When upgrades packages, many things have to be kept in mind. The following
|
|
is the understanding on how unattended-upgrades does its upgrades. These
|
|
precautions need to be taken into account whenever relevant.
|
|
|
|
Checks done by unattended upgrades before upgrading:
|
|
- Check if today is a configured patch day.
|
|
- Check if running on development release.
|
|
- Cache does not have broken packages.
|
|
- Whether system (and hence this process itself) is shutting down.
|
|
- System is on AC Power (if configured).
|
|
- Whether system is on a metered connection.
|
|
- Package system is locked.
|
|
- Check if dpkg journal is dirty and fix it.
|
|
- Take care that upgrade process does not terminate/crash self.
|
|
- Check if cache has broken dependencies.
|
|
- Remove auto removable packages.
|
|
- Download all packages before upgrade
|
|
- Perform conffile prompt checks
|
|
- Ignore if dpkg options has --force-confold or --force-confnew
|
|
|
|
Unattended upgrades checks for configuration file prompts as follows:
|
|
- Package downloaded successfully.
|
|
- Package is complete.
|
|
- File is available.
|
|
- Package is trusted.
|
|
- Package is not blacklisted.
|
|
- Package extension is .deb.
|
|
- Get conffiles values from control data of .deb.
|
|
- Read from /var/lib/dpkg/status, Package and Conffiles section.
|
|
- Read each conffile from system and compute md5sum.
|
|
- If new package md5sum == dpkg status md5 == file's md5 then no conffile
|
|
prompt.
|
|
- If conffiles in new package not found in dpkg status but found on system
|
|
with different md5sum, conffile prompt is available.
|
|
|
|
Filtering of packages to upgrade is done as follows:
|
|
- Packages is upgradable.
|
|
- Remove packages from blacklist
|
|
- Remove packages unless in whitelist or whitelist is empty
|
|
- Package in allowed origins
|
|
- Don't consider packages that require restart (if configured).
|
|
- Package dependencies satisfy origins, blacklist and whitelist.
|
|
- Order packages alphabetically (order changes when upgrading in minimal
|
|
steps).
|
|
|
|
The upgrade process looks like:
|
|
- Check for install while shutting down.
|
|
- Check for dry run.
|
|
- Mail list of changes.
|
|
- Upgrade in minimal steps.
|
|
- Clean downloaded packages.
|
|
|
|
"""
|
|
|
|
_run_lock = threading.Lock()
|
|
_wait_event = threading.Event()
|
|
|
|
UPGRADE_ATTEMPTS = 12
|
|
|
|
UPGRADE_ATTEMPT_WAIT_SECONDS = 30 * 60
|
|
|
|
class TemporaryFailure(Exception):
|
|
"""Raised when upgrade fails but can be tried again immediately."""
|
|
|
|
class PermanentFailure(Exception):
|
|
"""Raised when upgrade fails and there is nothing more we wish to do.
|
|
"""
|
|
|
|
def __init__(self):
|
|
"""Initialize the force upgrader."""
|
|
if plinth.cfg.develop:
|
|
self.UPGRADE_ATTEMPT_WAIT_SECONDS = 10
|
|
|
|
def on_package_cache_updated(self):
|
|
"""Trigger upgrades when notified about changes to package cache.
|
|
|
|
Call the upgrade process guaranteeing that it will not run more than
|
|
once simultaneously.
|
|
|
|
"""
|
|
if not self._run_lock.acquire(blocking=False):
|
|
logger.info('Package upgrade already in process')
|
|
return
|
|
|
|
try:
|
|
self._run_upgrade()
|
|
finally:
|
|
self._run_lock.release()
|
|
|
|
def _run_upgrade(self):
|
|
"""Attempt the upgrade process multiple times.
|
|
|
|
This method is guaranteed to not to run more than once simultaneously.
|
|
|
|
"""
|
|
for _ in range(self.UPGRADE_ATTEMPTS):
|
|
logger.info('Waiting for %s seconds before attempting upgrade',
|
|
self.UPGRADE_ATTEMPT_WAIT_SECONDS)
|
|
if self._wait_event.wait(self.UPGRADE_ATTEMPT_WAIT_SECONDS):
|
|
logger.info('Stopping upgrade attempts due to shutdown')
|
|
return
|
|
|
|
try:
|
|
logger.info('Attempting to perform upgrade')
|
|
self._attempt_upgrade()
|
|
return
|
|
except self.TemporaryFailure as exception:
|
|
logger.info('Cannot perform upgrade now: %s', exception)
|
|
except self.PermanentFailure as exception:
|
|
logger.error('Upgrade failed: %s', exception)
|
|
return
|
|
except Exception as exception:
|
|
# Assume all other errors are temporary
|
|
logger.exception('Unknown exception: %s', exception)
|
|
|
|
logger.info('Giving up on upgrade after too many retries')
|
|
|
|
def shutdown(self):
|
|
"""If we are sleeping for next attempt, cancel it.
|
|
|
|
If we are actually upgrading packages, don nothing.
|
|
"""
|
|
self._wait_event.set()
|
|
|
|
def _attempt_upgrade(self):
|
|
"""Attempt to perform an upgrade.
|
|
|
|
Raise TemporaryFailure if upgrade can't be performed now.
|
|
|
|
Raise PermanentFailure if upgrade can't be performed until something
|
|
with the system state changes. We don't want to try again until
|
|
notified of further package cache changes.
|
|
|
|
Return nothing otherwise.
|
|
|
|
"""
|
|
if _is_shutting_down:
|
|
raise self.PermanentFailure('Service is shutting down')
|
|
|
|
if package.is_package_manager_busy():
|
|
raise self.TemporaryFailure('Package manager is busy')
|
|
|
|
apps = self._get_list_of_apps_to_force_upgrade()
|
|
logger.info(
|
|
'Apps needing conffile upgrades: %s',
|
|
', '.join([str(app.app.info.name) for app in apps]) or 'None')
|
|
|
|
need_retry = False
|
|
for app, packages in apps.items():
|
|
try:
|
|
logger.info('Force upgrading app: %s', app.app.info.name)
|
|
if app.force_upgrade(app.setup_helper, packages):
|
|
logger.info('Successfully force upgraded app: %s',
|
|
app.app.info.name)
|
|
else:
|
|
logger.info('Ignored force upgrade for app: %s',
|
|
app.app.info.name)
|
|
except Exception as exception:
|
|
logger.exception('Error running force upgrade: %s', exception)
|
|
need_retry = True
|
|
# Continue trying to upgrade other apps
|
|
|
|
if need_retry:
|
|
raise self.TemporaryFailure('Some apps failed to force upgrade.')
|
|
|
|
def _get_list_of_apps_to_force_upgrade(self):
|
|
"""Return a list of app modules on which to run force upgrade."""
|
|
packages = self._get_list_of_upgradable_packages()
|
|
if not packages: # No packages to upgrade
|
|
return {}
|
|
|
|
package_names = [package.name for package in packages]
|
|
logger.info('Packages available for upgrade: %s',
|
|
', '.join(package_names))
|
|
|
|
managed_packages, package_apps_map = self._filter_managed_packages(
|
|
package_names)
|
|
if not managed_packages:
|
|
return {}
|
|
|
|
conffile_packages = package.filter_conffile_prompt_packages(
|
|
managed_packages)
|
|
logger.info('Packages that need conffile upgrades: %s',
|
|
conffile_packages)
|
|
|
|
apps = defaultdict(dict)
|
|
for package_name in conffile_packages:
|
|
for app in package_apps_map[package_name]:
|
|
apps[app][package_name] = conffile_packages[package_name]
|
|
|
|
return apps
|
|
|
|
@staticmethod
|
|
def _get_list_of_upgradable_packages():
|
|
"""Return list of packages that can be upgraded."""
|
|
cache = apt.cache.Cache()
|
|
return [package for package in cache if package.is_upgradable]
|
|
|
|
@staticmethod
|
|
def _filter_managed_packages(packages):
|
|
"""Filter out packages that apps don't force upgrade.
|
|
|
|
Return packages in the list that are managed by at least one app that
|
|
can perform force upgrade.
|
|
|
|
"""
|
|
package_apps_map = defaultdict(set)
|
|
upgradable_packages = set()
|
|
for module in plinth.module_loader.loaded_modules.values():
|
|
if not getattr(module, 'force_upgrade', None):
|
|
# App does not implement force upgrade
|
|
continue
|
|
|
|
if not _module_state_matches(module, 'up-to-date'):
|
|
# App is not installed.
|
|
# Or needs an update, let it update first.
|
|
continue
|
|
|
|
managed_packages = _get_module_managed_packages(module)
|
|
upgradable_packages.update(managed_packages)
|
|
|
|
for managed_package in managed_packages:
|
|
package_apps_map[managed_package].add(module)
|
|
|
|
return upgradable_packages.intersection(
|
|
set(packages)), package_apps_map
|
|
|
|
|
|
def on_package_cache_updated():
|
|
"""Called by D-Bus service when apt package cache is updated."""
|
|
global _force_upgrader
|
|
if not _force_upgrader:
|
|
_force_upgrader = ForceUpgrader()
|
|
|
|
_force_upgrader.on_package_cache_updated()
|