mirror of
https://github.com/freedombox/FreedomBox.git
synced 2026-07-29 12:09:37 +00:00
packages: Switch to installing with apt-get
- Use action helper so that Plinth can run unprivilaged and action script can run as root. - Use Status-Fd feature of apt-get to report progress. Don't report much detail. - Capture all stderr of the apt-get process and present it only in case of failure. - Remove package installation using PackageKit. Remove dependency on PackageKit. - Merge --setup package installation with regular package installation. This should fix the following problems: - PackageKit throws errors when APT encounters an error and later corrects them and proceeds well. This is reported upstream but not fixed. - PackageKit does not install recommends by default and there is no easy way to tell it to do so. - In some rare cases, PackageKit could get stuck for interactive input even though interactive flag is set to false. - PackageKit does not work without network manager connections. (Could have been mitigated by altering packagekit configuration). - PackageKit glib library leaks file descriptors after each operation. This leads to running out of fds during long running refresh operations such as OpenVPN setup. (This should have subsided by not checking package install with the new setup mechanism.)] Known issues: - In development mode, inside action scripts the python modules are always loaded from system path and not development directory. - With PackageKit it is possible to run multiple operations simultaneously. Others would wait while the first is being installed. With new implementation, the others error out unable to obtain lock.
This commit is contained in:
Sunil Mohan Adapa
James Valleroy
parent
5f548a9e36
commit
7a9a4cd861
2
INSTALL
2
INSTALL
@ -10,14 +10,12 @@
|
||||
gettext \
|
||||
gir1.2-glib-2.0 \
|
||||
gir1.2-networkmanager-1.0 \
|
||||
gir1.2-packagekitglib-1.0 \
|
||||
ldapscripts \
|
||||
libjs-bootstrap \
|
||||
libjs-jquery \
|
||||
libjs-modernizr \
|
||||
make \
|
||||
network-manager \
|
||||
packagekit \
|
||||
ppp \
|
||||
pppoe \
|
||||
python3 \
|
||||
|
||||
106
actions/packages
Executable file
106
actions/packages
Executable file
@ -0,0 +1,106 @@
|
||||
#!/usr/bin/python3
|
||||
#
|
||||
# This file is part of Plinth.
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as
|
||||
# published by the Free Software Foundation, either version 3 of the
|
||||
# License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
|
||||
"""
|
||||
Wrapper to handle package installation with apt-get.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from importlib import import_module
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from plinth import cfg
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
"""Return parsed command line arguments as dictionary."""
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest='subcommand', help='Sub command')
|
||||
|
||||
subparsers.add_parser('update', help='update the package lists')
|
||||
|
||||
subparser = subparsers.add_parser('install', help='install packages')
|
||||
subparser.add_argument(
|
||||
'module', help='name of module for which package is being installed')
|
||||
subparser.add_argument(
|
||||
'packages', nargs='+', help='list of packages to install')
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _run_apt_command(arguments):
|
||||
"""Run apt-get with provided arguments."""
|
||||
# Ask apt-get to output its progress to file descriptor 3.
|
||||
command = ['apt-get', '--assume-yes', '--quiet=2', '--option',
|
||||
'APT::Status-Fd=3'] + arguments
|
||||
|
||||
# Duplicate stdout to file descriptor 3 for this process.
|
||||
os.dup2(1, 3)
|
||||
|
||||
# Pass on file descriptor 3 instead of closing it. Close stdout
|
||||
# so that regular output is ignored.
|
||||
env = os.environ.copy()
|
||||
env['DEBIAN_FRONTEND'] = 'noninteractive'
|
||||
process = subprocess.run(
|
||||
command, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
||||
close_fds=False, env=env)
|
||||
sys.exit(process.returncode)
|
||||
|
||||
|
||||
def subcommand_update(arguments):
|
||||
"""Update apt package lists."""
|
||||
_run_apt_command(['update'])
|
||||
|
||||
|
||||
def subcommand_install(arguments):
|
||||
"""Install packages using apt-get."""
|
||||
try:
|
||||
_assert_managed_packages(arguments.module, arguments.packages)
|
||||
except Exception as exception:
|
||||
print('Access check failed:', exception, file=sys.stderr)
|
||||
sys.exit(99)
|
||||
|
||||
_run_apt_command(['install'] + arguments.packages)
|
||||
|
||||
|
||||
def _assert_managed_packages(module, packages):
|
||||
"""Check that list of packages are in fact managed by module."""
|
||||
cfg.read()
|
||||
module_file = os.path.join(cfg.config_dir, 'modules-enabled', module)
|
||||
|
||||
with open(module_file, 'r') as file_handle:
|
||||
module_path = file_handle.read().strip()
|
||||
|
||||
module = import_module(module_path)
|
||||
for package in packages:
|
||||
assert package in module.managed_packages
|
||||
|
||||
|
||||
def main():
|
||||
"""Parse arguments and perform all duties."""
|
||||
arguments = parse_arguments()
|
||||
|
||||
subcommand = arguments.subcommand.replace('-', '_')
|
||||
subcommand_method = globals()['subcommand_' + subcommand]
|
||||
subcommand_method(arguments)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -22,15 +22,12 @@ Framework for installing and updating distribution packages
|
||||
from django.utils.translation import ugettext as _
|
||||
import logging
|
||||
import subprocess
|
||||
import threading
|
||||
|
||||
from plinth.utils import import_from_gi
|
||||
glib = import_from_gi('GLib', '2.0')
|
||||
packagekit = import_from_gi('PackageKitGlib', '1.0')
|
||||
from plinth import actions
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
transactions = {}
|
||||
packages_resolved = {}
|
||||
|
||||
|
||||
class PackageException(Exception):
|
||||
@ -52,128 +49,25 @@ class PackageException(Exception):
|
||||
class Transaction(object):
|
||||
"""Information about an ongoing transaction."""
|
||||
|
||||
def __init__(self, package_names):
|
||||
def __init__(self, module_name, package_names):
|
||||
"""Initialize transaction object.
|
||||
|
||||
Set most values to None until they are sent as progress update.
|
||||
"""
|
||||
self.module_name = module_name
|
||||
self.package_names = package_names
|
||||
|
||||
# Progress
|
||||
self.allow_cancel = None
|
||||
self.percentage = None
|
||||
self.status = None
|
||||
self.status_string = None
|
||||
self.flags = None
|
||||
self.package = None
|
||||
self.package_id = None
|
||||
self.item_progress = None
|
||||
self.role = None
|
||||
self.caller_active = None
|
||||
self.download_size_remaining = None
|
||||
self.speed = None
|
||||
self._reset_status()
|
||||
|
||||
def get_id(self):
|
||||
"""Return a identifier to use as a key in a map of transactions."""
|
||||
return frozenset(self.package_names)
|
||||
|
||||
def __str__(self):
|
||||
"""Return the string representation of the object"""
|
||||
return ('Transaction(packages={0}, allow_cancel={1}, status={2}, '
|
||||
' percentage={3}, package={4}, item_progress={5})').format(
|
||||
self.package_names, self.allow_cancel, self.status_string,
|
||||
self.percentage, self.package, self.item_progress)
|
||||
|
||||
def install(self):
|
||||
"""Run a PackageKit transaction to install given packages."""
|
||||
try:
|
||||
self._do_install()
|
||||
except glib.Error as exception:
|
||||
raise PackageException(exception.message) from exception
|
||||
|
||||
def _do_install(self):
|
||||
"""Run a PackageKit transaction to install given packages.
|
||||
|
||||
Raise exception in case of error.
|
||||
"""
|
||||
client = packagekit.Client()
|
||||
client.set_interactive(False)
|
||||
|
||||
# Refresh package cache from all enabled repositories
|
||||
results = client.refresh_cache(
|
||||
False, None, self.progress_callback, self)
|
||||
self._assert_success(results)
|
||||
|
||||
# Resolve packages again to get the latest versions after refresh
|
||||
results = client.resolve(packagekit.FilterEnum.INSTALLED,
|
||||
tuple(self.package_names) + (None, ),
|
||||
None, self.progress_callback, self)
|
||||
self._assert_success(results)
|
||||
|
||||
for package in results.get_package_array():
|
||||
packages_resolved[package.get_name()] = package
|
||||
|
||||
package_ids = []
|
||||
for package_name in self.package_names:
|
||||
if package_name not in packages_resolved or \
|
||||
not packages_resolved[package_name]:
|
||||
raise PackageException(_('packages not found'))
|
||||
|
||||
package_ids.append(packages_resolved[package_name].get_id())
|
||||
|
||||
# Start package installation
|
||||
results = client.install_packages(
|
||||
packagekit.TransactionFlagEnum.ONLY_TRUSTED, package_ids + [None],
|
||||
None, self.progress_callback, self)
|
||||
self._assert_success(results)
|
||||
|
||||
def _assert_success(self, results):
|
||||
"""Check that the most recent operation was a success."""
|
||||
if results and results.get_error_code() is not None:
|
||||
error = results.get_error_code()
|
||||
error_code = error.get_code() if error else None
|
||||
error_string = packagekit.ErrorEnum.to_string(error_code) \
|
||||
if error_code else None
|
||||
error_details = error.get_details() if error else None
|
||||
raise PackageException(error_string, error_details)
|
||||
|
||||
def progress_callback(self, progress, progress_type, user_data):
|
||||
"""Process progress updates on package resolve operation"""
|
||||
if progress_type == packagekit.ProgressType.PERCENTAGE:
|
||||
self.percentage = progress.props.percentage
|
||||
elif progress_type == packagekit.ProgressType.PACKAGE:
|
||||
self.package = progress.props.package
|
||||
elif progress_type == packagekit.ProgressType.ALLOW_CANCEL:
|
||||
self.allow_cancel = progress.props.allow_cancel
|
||||
elif progress_type == packagekit.ProgressType.PACKAGE_ID:
|
||||
self.package_id = progress.props.package_id
|
||||
elif progress_type == packagekit.ProgressType.ITEM_PROGRESS:
|
||||
self.item_progress = progress.props.item_progress
|
||||
elif progress_type == packagekit.ProgressType.STATUS:
|
||||
self.status = progress.props.status
|
||||
self.status_string = \
|
||||
packagekit.StatusEnum.to_string(progress.props.status)
|
||||
elif progress_type == packagekit.ProgressType.TRANSACTION_FLAGS:
|
||||
self.flags = progress.props.transaction_flags
|
||||
elif progress_type == packagekit.ProgressType.ROLE:
|
||||
self.role = progress.props.role
|
||||
elif progress_type == packagekit.ProgressType.CALLER_ACTIVE:
|
||||
self.caller_active = progress.props.caller_active
|
||||
elif progress_type == packagekit.ProgressType.DOWNLOAD_SIZE_REMAINING:
|
||||
self.download_size_remaining = \
|
||||
progress.props.download_size_remaining
|
||||
elif progress_type == packagekit.ProgressType.SPEED:
|
||||
self.speed = progress.props.speed
|
||||
else:
|
||||
logger.info('Unhandle packagekit progress callback - %s, %s',
|
||||
progress, progress_type)
|
||||
|
||||
|
||||
class AptTransaction(object):
|
||||
"""Install a package using Apt."""
|
||||
def __init__(self, package_names):
|
||||
"""Initialize transaction object."""
|
||||
self.package_names = package_names
|
||||
def _reset_status(self):
|
||||
"""Reset the current status progress."""
|
||||
self.status_string = ''
|
||||
self.percentage = 0
|
||||
self.stderr = None
|
||||
|
||||
def install(self):
|
||||
"""Run a PackageKit transaction to install given packages.
|
||||
@ -183,9 +77,57 @@ class AptTransaction(object):
|
||||
when --setup is argument is passed.
|
||||
"""
|
||||
try:
|
||||
subprocess.run(['apt-get', 'update'])
|
||||
subprocess.run(['apt-get', '-y', 'install'] + self.package_names,
|
||||
check=True)
|
||||
self._run_apt_command(['update'])
|
||||
self._run_apt_command(['install', self.module_name] +
|
||||
self.package_names)
|
||||
except subprocess.CalledProcessError as exception:
|
||||
logger.exception('Error installing package: %s', exception)
|
||||
raise
|
||||
|
||||
def _run_apt_command(self, arguments):
|
||||
"""Run apt-get and update progress."""
|
||||
self._reset_status()
|
||||
|
||||
process = actions.superuser_run('packages', arguments, async=True)
|
||||
process.stdin.close()
|
||||
|
||||
stdout_thread = threading.Thread(target=self._read_stdout,
|
||||
args=(process,))
|
||||
stderr_thread = threading.Thread(target=self._read_stderr,
|
||||
args=(process,))
|
||||
stdout_thread.start()
|
||||
stderr_thread.start()
|
||||
stdout_thread.join()
|
||||
stderr_thread.join()
|
||||
|
||||
return_code = process.wait()
|
||||
if return_code != 0:
|
||||
raise PackageException(_('Error during installation'), self.stderr)
|
||||
|
||||
def _read_stdout(self, process):
|
||||
"""Read the stdout of the process and update progress."""
|
||||
for line in process.stdout:
|
||||
self._parse_progress(line.decode())
|
||||
|
||||
def _read_stderr(self, process):
|
||||
"""Read the stderr of the process and store in buffer."""
|
||||
self.stderr = process.stderr.read().decode()
|
||||
|
||||
def _parse_progress(self, line):
|
||||
"""Parse the apt-get process output line.
|
||||
|
||||
See README.progress-reporting in apt source code.
|
||||
"""
|
||||
parts = line.split(':')
|
||||
if len(parts) < 4:
|
||||
return
|
||||
|
||||
status_map = {
|
||||
'pmstatus': _('installing'),
|
||||
'dlstatus': _('downloading'),
|
||||
'media-change': _('media change'),
|
||||
'pmconffile': _('configuration file: {file}').format(
|
||||
file=parts[1]),
|
||||
}
|
||||
self.status_string = status_map.get(parts[0], '')
|
||||
self.percentage = int(float(parts[2]))
|
||||
|
||||
@ -27,8 +27,6 @@ import plinth
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
running_initial_setup = False
|
||||
|
||||
|
||||
class Helper(object):
|
||||
"""Helper routines for modules to show progress."""
|
||||
@ -93,17 +91,12 @@ class Helper(object):
|
||||
"""Install a set of packages marking progress."""
|
||||
logger.info('Running install for module - %s, packages - %s',
|
||||
self.module_name, package_names)
|
||||
if running_initial_setup:
|
||||
transaction = package.AptTransaction(package_names)
|
||||
transaction.install()
|
||||
return
|
||||
|
||||
transaction = package.Transaction(package_names)
|
||||
transaction = package.Transaction(self.module_name, package_names)
|
||||
self.current_operation = {
|
||||
'step': 'install',
|
||||
'transaction': transaction,
|
||||
}
|
||||
|
||||
transaction.install()
|
||||
|
||||
def call(self, step, method, *args, **kwargs):
|
||||
@ -160,8 +153,6 @@ def init(module_name, module):
|
||||
def setup_all_modules(essential=False):
|
||||
"""Run setup on all essential modules and exit."""
|
||||
logger.info('Running setup for all modules, essential - %s', essential)
|
||||
global running_initial_setup
|
||||
running_initial_setup = True
|
||||
for module_name, module in plinth.module_loader.loaded_modules.items():
|
||||
if essential and not getattr(module, 'is_essential', False):
|
||||
continue
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user