FreedomBox/actions/upgrades

#!/usr/bin/python3
# SPDX-License-Identifier: AGPL-3.0-or-later
"""
Configures or runs unattended-upgrades
"""

import argparse
import json
import logging
import os
import pathlib
import re
import subprocess
import sys
import time

from plinth.action_utils import (apt_hold, apt_hold_flag, apt_hold_freedombox,
                                 apt_unhold_freedombox, debconf_set_selections,
                                 run_apt_command, service_daemon_reload,
                                 service_restart)
from plinth.modules.apache.components import check_url
from plinth.modules.snapshot import (is_apt_snapshots_enabled, is_supported as
                                     snapshot_is_supported, load_augeas as
                                     snapshot_load_augeas)
from plinth.modules.upgrades import (BACKPORTS_SOURCES_LIST, SOURCES_LIST,
                                     get_current_release, is_backports_current)

AUTO_CONF_FILE = '/etc/apt/apt.conf.d/20auto-upgrades'
LOG_FILE = '/var/log/unattended-upgrades/unattended-upgrades.log'
DPKG_LOG_FILE = '/var/log/unattended-upgrades/unattended-upgrades-dpkg.log'
RELEASE_FILE_URL = \
    'https://deb.debian.org/debian/dists/{}/Release'

APT_PREFERENCES_FREEDOMBOX = '''Explanation: This file is managed by FreedomBox, do not edit.
Explanation: Allow carefully selected updates to 'freedombox' from backports.
Package: freedombox
Pin: release a={}-backports
Pin-Priority: 500
'''

# Whenever these preferences needs to change, increment the version number
# upgrades app. This ensures that setup is run again and the new contents are
# overwritten on the old file.
APT_PREFERENCES_APPS = '''Explanation: This file is managed by FreedomBox, do not edit.
Explanation: matrix-synapse 0.99.5 introduces room version 4. Older version
Explanation: 0.99.2 in buster won't be able join newly created rooms.
Package: matrix-synapse
Pin: release a=buster-backports
Pin-Priority: 500

Explanation: matrix-synapse >= 1.2 requires python3-service-identity >= 18.1
Package: python3-service-identity
Pin: release a=buster-backports
Pin-Priority: 500

Explanation: matrix-synapse >= 1.5 requires python3-typing-extensions >= 3.7.4
Package: python3-typing-extensions
Pin: release a=buster-backports
Pin-Priority: 500

Explanation: matrix-synapse >= 1.11 requires python3-signedjson >= 1.1.0
Package: python3-signedjson
Pin: release a=buster-backports
Pin-Priority: 500

Explanation: matrix-synapse >= 1.12 requires python3-twisted >= 18.9.0-8~
Package: python3-twisted
Pin: release a=buster-backports
Pin-Priority: 500

Explanation: python3-twisted requires matching version of python3-twisted-bin
Package: python3-twisted-bin
Pin: release a=buster-backports
Pin-Priority: 500

Explanation: matrix-synapse >= 1.16 requires python3-attr >= 19.1.0~
Package: python3-attr
Pin: release a=buster-backports
Pin-Priority: 500

Explanation: matrix-synapse >= 1.19 requires python3-canonicaljson >= 1.2.0
Package: python3-canonicaljson
Pin: release a=buster-backports
Pin-Priority: 500

Explanation: matrix-synapse >= 1.26 recommends python3-psycopg2 >= 2.8
Package: python3-psycopg2
Pin: release a=buster-backports
Pin-Priority: 500
'''

DIST_UPGRADE_SERVICE = '''
[Unit]
Description=Upgrade to new stable Debian release

[Service]
Type=oneshot
ExecStart=/usr/share/plinth/actions/upgrades dist-upgrade
KillMode=process
TimeoutSec=12hr
'''

DIST_UPGRADE_SERVICE_PATH = \
    '/run/systemd/system/freedombox-dist-upgrade.service'

dist_upgrade_flag = pathlib.Path(
    '/var/lib/freedombox/dist-upgrade-in-progress')


def parse_arguments():
    """Return parsed command line arguments as dictionary"""
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers(dest='subcommand', help='Sub command')

    subparsers.add_parser('run', help='Upgrade packages on the system')
    subparsers.add_parser('check-auto',
                          help='Check if automatic upgrades are enabled')
    subparsers.add_parser('enable-auto', help='Enable automatic upgrades')
    subparsers.add_parser('disable-auto', help='Disable automatic upgrades.')
    subparsers.add_parser('get-log', help='Print the automatic upgrades log')

    subparsers.add_parser('setup', help='Setup apt preferences')

    activate_backports = subparsers.add_parser(
        'activate-backports', help='Activate backports if possible')
    activate_backports.add_argument('--develop', required=False, default=False,
                                    action='store_true',
                                    help='Development mode')

    start_dist_upgrade = subparsers.add_parser(
        'start-dist-upgrade', help='Check and start dist upgrade process')
    start_dist_upgrade.add_argument(
        '--test', required=False, default=False, action='store_true',
        help='Test dist-upgrade from stable to testing')
    subparsers.add_parser('dist-upgrade', help='Perform dist upgrade')

    subparsers.required = True
    return parser.parse_args()


def _run():
    """Run unattended-upgrades"""
    subprocess.run(['dpkg', '--configure', '-a'])
    run_apt_command(['--fix-broken', 'install'])

    # In case freedombox package was left in held state by an
    # interrupted process, release it.
    if apt_hold_flag.exists():
        apt_unhold_freedombox()

    subprocess.Popen(['systemctl', 'start', 'freedombox-manual-upgrade'],
                     stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
                     stderr=subprocess.DEVNULL, close_fds=True,
                     start_new_session=True)


def subcommand_run(_):
    """Run unattended-upgrades"""
    try:
        _run()
    except FileNotFoundError:
        print('Error: systemctl is not available.', file=sys.stderr)
        sys.exit(2)
    except Exception as error:
        print('Error: {0}'.format(error), file=sys.stderr)
        sys.exit(3)


def _check_auto():
    """Check if automatic upgrades are enabled."""
    arguments = [
        'apt-config', 'shell', 'UpdateInterval',
        'APT::Periodic::Update-Package-Lists'
    ]
    output = subprocess.check_output(arguments).decode()
    update_interval = 0
    match = re.match(r"UpdateInterval='(.*)'", output)
    if match:
        update_interval = int(match.group(1))

    return bool(update_interval)


def subcommand_check_auto(_):
    """Check if automatic upgrades are enabled"""
    try:
        print(_check_auto())
    except subprocess.CalledProcessError as error:
        print('Error: {0}'.format(error), file=sys.stderr)
        sys.exit(1)


def subcommand_enable_auto(_):
    """Enable automatic upgrades"""
    with open(AUTO_CONF_FILE, 'w') as conffile:
        conffile.write('APT::Periodic::Update-Package-Lists "1";\n')
        conffile.write('APT::Periodic::Unattended-Upgrade "1";\n')


def subcommand_disable_auto(_):
    """Disable automatic upgrades"""
    with open(AUTO_CONF_FILE, 'w') as conffile:
        conffile.write('APT::Periodic::Update-Package-Lists "0";\n')
        conffile.write('APT::Periodic::Unattended-Upgrade "0";\n')


def subcommand_get_log(_):
    """Print the automatic upgrades log."""
    try:
        print('==> ' + os.path.basename(LOG_FILE))
        with open(LOG_FILE, 'r') as file_handle:
            print(file_handle.read())
    except IOError:
        pass

    try:
        print('==> ' + os.path.basename(DPKG_LOG_FILE))
        with open(DPKG_LOG_FILE, 'r') as file_handle:
            print(file_handle.read())
    except IOError:
        pass


def _get_protocol():
    """Return the protocol to use for newly added repository sources."""
    try:
        from plinth.modules.tor import utils
        if utils.is_apt_transport_tor_enabled():
            return 'tor+http'
    except Exception:
        pass

    return 'http'


def _is_release_file_available(protocol, dist, backports=False):
    """Return whether the release for dist[-backports] is available."""
    wrapper = None
    if protocol == 'tor+http':
        wrapper = 'torsocks'

    if backports:
        dist += '-backports'

    result = check_url(RELEASE_FILE_URL.format(dist), wrapper=wrapper)
    return result == 'passed'


def _add_backports_sources(sources_list, protocol, dist):
    """Add backports sources to freedombox repositories list."""
    sources = '''# This file is managed by FreedomBox, do not edit.
# Allow carefully selected updates to 'freedombox' from backports.

deb {protocol}://deb.debian.org/debian {dist}-backports main
deb-src {protocol}://deb.debian.org/debian {dist}-backports main
'''
    sources = sources.format(protocol=protocol, dist=dist)
    with open(sources_list, 'w') as file_handle:
        file_handle.write(sources)


def _check_and_backports_sources(develop=False):
    """Add backports sources after checking if it is available."""
    old_sources_list = '/etc/apt/sources.list.d/freedombox.list'
    if os.path.exists(old_sources_list):
        os.remove(old_sources_list)

    if is_backports_current():
        print('Repositories list up-to-date. Skipping update.')
        return

    try:
        with open('/etc/dpkg/origins/default', 'r') as default_origin:
            matches = [
                re.match(r'Vendor:\s+Debian', line, flags=re.IGNORECASE)
                for line in default_origin.readlines()
            ]
    except FileNotFoundError:
        print('Could not open /etc/dpkg/origins/default')
        return

    if not any(matches):
        print('System is running a derivative of Debian. Skip enabling '
              'backports.')
        return

    release, dist = get_current_release()
    if release == 'unstable' or (release == 'testing' and not develop):
        print(f'System release is {release}. Skip enabling backports.')
        return

    protocol = _get_protocol()
    if protocol == 'tor+http':
        print('Package download over Tor is enabled.')

    if not _is_release_file_available(protocol, dist, backports=True):
        print(f'Release file for {dist}-backports is not available yet.')
        return

    print(f'{dist}-backports is now available. Adding to sources.')
    _add_backports_sources(BACKPORTS_SOURCES_LIST, protocol, dist)
    # In case of dist upgrade, rewrite the preferences file.
    _add_apt_preferences()


def _add_apt_preferences():
    """Setup APT preferences to upgrade selected packages from backports."""
    base_path = pathlib.Path('/etc/apt/preferences.d')
    for file_name in ['50freedombox.pref', '50freedombox2.pref']:
        full_path = base_path / file_name
        if full_path.exists():
            full_path.unlink()

    # Don't try to remove 50freedombox3.pref as this file is shipped with the
    # Debian package and is removed using maintainer scripts.

    _, dist = get_current_release()
    if dist == 'sid':
        print(f'System distribution is {dist}. Skip setting apt preferences '
              'for backports.')
    else:
        print(f'Setting apt preferences for {dist}-backports.')
        with open(base_path / '50freedombox4.pref', 'w') as file_handle:
            file_handle.write(APT_PREFERENCES_FREEDOMBOX.format(dist))
        with open(base_path / '51freedombox-apps.pref', 'w') as file_handle:
            file_handle.write(APT_PREFERENCES_APPS)


def _check_dist_upgrade(test_upgrade=False):
    """Check for new stable release, if updates are enabled, and if there is
    enough free space for the dist upgrade.

    If test_upgrade is True, also check for upgrade to testing.

    Returns (boolean, string) indicating if the upgrade is ready, and a reason
    if not.
    """
    if dist_upgrade_flag.exists():
        return (True, 'found-previous')

    release, dist = get_current_release()
    if release in ['unstable', 'testing']:
        return (False, f'already-{release}')

    check_dists = ['stable']
    if test_upgrade:
        check_dists.append('testing')

    codename = None
    for check_dist in check_dists:
        url = RELEASE_FILE_URL.format(check_dist)
        command = ['curl', '--silent', '--location', '--fail', url]
        protocol = _get_protocol()
        if protocol == 'tor+http':
            command.insert(0, 'torsocks')
            logging.info('Package download over Tor is enabled.')

        try:
            output = subprocess.check_output(command).decode()
        except (subprocess.CalledProcessError, FileNotFoundError):
            logging.warning('Error while checking for new %s release',
                            check_dist)
        else:
            for line in output.split('\n'):
                if line.startswith('Codename:'):
                    codename = line.split()[1]

    if not codename:
        return (False, 'codename-not-found')

    if codename == dist:
        return (False, f'already-{dist}')

    if not _check_auto():
        return (False, 'upgrades-not-enabled')

    if check_dist == 'testing' and not test_upgrade:
        return (False, 'test-not-set')

    output = subprocess.check_output(['df', '--output=avail', '/'])
    free_space = int(output.decode().split('\n')[1])
    if free_space < 5000000:
        return (False, 'not-enough-free-space')

    logging.info('Upgrading from %s to %s...', dist, codename)
    with open(SOURCES_LIST, 'r') as sources_list:
        lines = sources_list.readlines()

    with open(SOURCES_LIST, 'w') as sources_list:
        for line in lines:
            # E.g. replace 'buster' with 'bullseye'.
            new_line = line.replace(dist, codename)
            if check_dist == 'testing':
                # E.g. replace 'stable' with 'bullseye'.
                new_line = new_line.replace('stable', codename)

            # Security suite name renamed starting with bullseye
            if 'security' in new_line:
                new_line = new_line.replace('/updates', '-security')

            sources_list.write(new_line)

    logging.info('Dist upgrade in progress. Setting flag.')
    dist_upgrade_flag.touch(mode=0o660)
    return (True, 'started-dist-upgrade')


def _perform_dist_upgrade():
    """Perform upgrade to next release of Debian."""
    # Take a snapshot if supported and enabled, then disable snapshots.
    snapshots_supported = snapshot_is_supported()
    if snapshots_supported:
        print('Taking a snapshot before dist upgrade...', flush=True)
        subprocess.run(['/usr/share/plinth/actions/snapshot', 'create'],
                       check=True)
        aug = snapshot_load_augeas()
        apt_snapshots_enabled = is_apt_snapshots_enabled(aug)
        if apt_snapshots_enabled:
            print('Disable apt snapshots during dist upgrade...', flush=True)
            subprocess.run([
                '/usr/share/plinth/actions/snapshot', 'disable-apt-snapshot',
                'yes'
            ], check=True)
        else:
            print('Apt snapshots already disabled.', flush=True)
    else:
        print('Snapshots are not supported, skip taking a snapshot.',
              flush=True)

    # If searx is enabled, disable it until we can upgrade it properly.
    searx_is_enabled = pathlib.Path(
        '/etc/uwsgi/apps-enabled/searx.ini').exists()
    if searx_is_enabled:
        print('Disabling searx...', flush=True)
        subprocess.run([
            '/usr/share/plinth/actions/apache', 'uwsgi-disable', '--name',
            'searx'
        ], check=True)

    # Hold freedombox package during entire dist upgrade.
    print('Holding freedombox package...', flush=True)
    with apt_hold_freedombox():
        print('Updating Apt cache...', flush=True)
        run_apt_command(['update'])

        print('Upgrading base-files and unattended-upgrades...', flush=True)
        run_apt_command(['install', 'base-files'])
        run_apt_command(['install', 'python3-systemd', 'unattended-upgrades'])

        # Tell grub-pc to continue without installing grub again.
        print('Set grub-pc to not require re-installing grub...', flush=True)
        debconf_set_selections(
            ['grub-pc grub-pc/install_devices_empty boolean true'])

        print('Running unattended-upgrade...', flush=True)
        subprocess.run(['unattended-upgrade', '--verbose'])

        # Remove obsolete packages that may prevent other packages from
        # upgrading.
        print('Removing libgcc1...', flush=True)
        run_apt_command(['remove', 'libgcc1'])

        # Hold packages known to have conffile prompts. FreedomBox service
        # will handle their upgrade later.
        packages_with_prompts = [
            'firewalld', 'mumble-server', 'radicale', 'roundcube-core'
        ]
        print(
            'Holding packages with conffile prompts: ' +
            ', '.join(packages_with_prompts) + '...', flush=True)
        with apt_hold(packages_with_prompts):
            print('Holding tt-rss package if available...', flush=True)
            with apt_hold(['tt-rss'], ignore_errors=True):
                print('Running apt full-upgrade...', flush=True)
                run_apt_command(['full-upgrade'])

        # If searx is installed, update search engines list.
        if pathlib.Path('/etc/searx/settings.yml').exists():
            print('Updating searx search engines list...', flush=True)
            subprocess.run(['/usr/share/plinth/actions/searx', 'setup'],
                           check=True)
            if searx_is_enabled:
                print('Re-enabling searx after upgrade...', flush=True)
                subprocess.run([
                    '/usr/share/plinth/actions/apache', 'uwsgi-enable',
                    '--name', 'searx'
                ], check=True)

        print('Running apt autoremove...', flush=True)
        run_apt_command(['autoremove'])

    # Run unattended-upgrade once more to handle upgrading the
    # freedombox package.
    print('Running unattended-upgrade...', flush=True)
    subprocess.run(['unattended-upgrade', '--verbose'])

    # Restore original snapshots configuration.
    if snapshots_supported and apt_snapshots_enabled:
        print('Re-enable apt snapshots...', flush=True)
        subprocess.run([
            '/usr/share/plinth/actions/snapshot', 'disable-apt-snapshot', 'no'
        ], check=True)

    # Restart FreedomBox service to ensure it is using the latest
    # dependencies.
    print('Restarting FreedomBox service...', flush=True)
    service_restart('plinth')

    # After 10 minutes, update apt cache again to trigger force_upgrades.
    print('Waiting for 10 minutes...', flush=True)
    time.sleep(10 * 60)
    print('Updating Apt cache...', flush=True)
    run_apt_command(['update'])

    print('Dist upgrade complete. Removing flag.', flush=True)
    if dist_upgrade_flag.exists():
        dist_upgrade_flag.unlink()


def subcommand_setup(_):
    """Setup apt preferences."""
    _add_apt_preferences()


def subcommand_activate_backports(arguments):
    """Setup software repositories needed for FreedomBox.

    Repositories list for now only contains the backports. If the file exists,
    assume that it contains backports.

    """
    _check_and_backports_sources(arguments.develop)


def subcommand_start_dist_upgrade(arguments):
    """Start dist upgrade process.

    Check if a new stable release is available, and start dist-upgrade process
    if updates are enabled.
    """
    upgrade_ready, reason = _check_dist_upgrade(arguments.test)
    if upgrade_ready:
        with open(DIST_UPGRADE_SERVICE_PATH, 'w') as service_file:
            service_file.write(DIST_UPGRADE_SERVICE)

        service_daemon_reload()
        subprocess.Popen(['systemctl', 'start', 'freedombox-dist-upgrade'],
                         stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
                         stderr=subprocess.DEVNULL, close_fds=True,
                         start_new_session=True)

    print(
        json.dumps({
            'dist_upgrade_started': upgrade_ready,
            'reason': reason,
        }))


def subcommand_dist_upgrade(_):
    """Perform major distribution upgrade.
    """
    _perform_dist_upgrade()


def main():
    """Parse arguments and perform all duties"""
    arguments = parse_arguments()

    subcommand = arguments.subcommand.replace('-', '_')
    subcommand_method = globals()['subcommand_' + subcommand]
    subcommand_method(arguments)


if __name__ == '__main__':
    main()