package: Refresh apt cache if old and some packages are not found

Fixes: #1358

- Refresh the apt cache if required packages for an app are not found and if the
cache is more than 1 hour old (or non-existent).

- If required packages are found, don't refresh the package cache even if the
cache is outdated. This is because the check operation could lead to many
minutes of waiting before app can be installed.

Tests:

- Remove /var/lib/apt/lists/* and /var/cache/apt/pkgcache.bin. Visit an app
setup page. apt cache is updated and it take a while to check that the app is
available. App is shown as available. If page is refreshed, this time, the cache
is not updated.

- Set the modification of /var/cache/apt/pkgcache.bin file to more than 2 hours
ago with 'touch -d "2 hours ago" /var/cache/apt/pkgcache.bin'. Then refreshing
the page will not refresh the cache.

- Repeat test with an app that is not available such as Janus. Again apt cache
is refreshed. App is shown as not available. On refresh, the cache is not
updated.

- Set the modification of /var/cache/apt/pkgcache.bin file to more than 2 hours
ago with 'touch -d "2 hours ago" /var/cache/apt/pkgcache.bin'. Then refreshing
the page will not refresh the cache.

Signed-off-by: Sunil Mohan Adapa <sunil@medhas.org>
Reviewed-by: Joseph Nuthalapati <njoseph@riseup.net>

- Remove redundant if condition in setup.html template
- Use JavaScript fetch() API instead of XMLHTTPRequest class
- Update a comment in test_package.py
Signed-off-by: Joseph Nuthalapati <njoseph@riseup.net>

plinth/package.py

@@ -255,14 +255,26 @@ class Packages(app_module.FollowerComponent):
         Returns True if all of the packages are available in the user's Debian
         distribution or False otherwise. Returns True if it cannot be reliably
         determined whether the packages are available or not.
-        """
-        apt_lists_dir = pathlib.Path('/var/lib/apt/lists/')
-        num_files = len(
-            [child for child in apt_lists_dir.iterdir() if child.is_file()])
-        if num_files < 2:  # not counting the lock file
-            return True  # Don't know, package cache is not available
 
-        # List of all packages from all Package components
+        This operation can run for a long time as package cache may be
+        refreshed.
+        """
+        try:
+            self.get_actual_packages()
+            return True  # All packages were found in the cache
+        except MissingPackageError:
+            pass  # We will retry after refreshing package list
+
+        # If the package cache is new, then package is really not available.
+        package_cache = pathlib.Path('/var/cache/apt/pkgcache.bin')
+        if (package_cache.exists()
+                and time.time() - package_cache.stat().st_mtime < 3600):
+            return False
+
+        # Perform 'apt-get update'
+        refresh_package_lists()
+
+        # Check again for all the packages
         try:
             self.get_actual_packages()
         except MissingPackageError:

plinth/templates/setup.html

@@ -62,8 +62,7 @@
     <form class="form-install" action="" method="post">
       {% csrf_token %}
 
-      {% if has_unavailable_packages %}
-      {% elif package_conflicts and package_conflicts_action.value != 'ignore' %}
+      {% if package_conflicts and package_conflicts_action.value != 'ignore' %}
         <div class="alert alert-warning d-flex align-items-center" role="alert">
           <div class="me-2">
             <span class="fa fa-exclamation-triangle" aria-hidden="true"></span>

plinth/tests/test_package.py

@@ -3,6 +3,7 @@
 Test module for package module.
 """
 
+import time
 import unittest
 from unittest.mock import Mock, call, patch
 
@@ -338,23 +339,46 @@ def test_packages_find_conflicts(packages_installed_):
     assert component.find_conflicts() == ['package1', 'package2']
 
 
+@patch('plinth.package.refresh_package_lists')
 @patch('apt.Cache')
 @patch('pathlib.Path')
-def test_packages_is_available(path_class, cache):
+def test_packages_is_available(path_class, cache, refresh_package_lists):
     """Test checking for available packages."""
     path = Mock()
     path_class.return_value = path
-    path.iterdir.return_value = [Mock()]
 
+    # Packages found in cache
     component = Packages('test-component', ['package1', 'package2'])
-    assert component.is_available()
-
-    path.iterdir.return_value = [Mock(), Mock()]
     cache.return_value = ['package1', 'package2']
     assert component.is_available()
+    path_class.assert_not_called()
+    refresh_package_lists.assert_not_called()
 
+    # Packages not found, cache exists and is fresh
     cache.return_value = ['package1']
+    path.exists.return_value = True
+    path.stat.return_value.st_mtime = time.time()
     assert not component.is_available()
+    refresh_package_lists.assert_not_called()
+
+    # Packages not found, cache does not exist
+    cache.return_value = ['package1']
+    path.exists.return_value = False
+    assert not component.is_available()
+    refresh_package_lists.assert_called_once()
+
+    # Packages not found, cache is stale
+    cache.return_value = ['package1']
+    refresh_package_lists.reset_mock()
+    path.exists.return_value = True
+    path.stat.return_value.st_mtime = time.time() - 7200
+    assert not component.is_available()
+    refresh_package_lists.assert_called_once()
+
+    # Packages not found, cache is stale, but packages found after refresh
+    cache.side_effect = [['package1'], ['package1', 'package2']]
+    refresh_package_lists.reset_mock()
+    assert component.is_available()
 
 
 def test_packages_installed():

static/themes/default/js/main.js

@@ -210,7 +210,7 @@ function setSelectAllValue(parent) {
 /*
  * Check whether an app is available on its setup page.
  */
-document.addEventListener('DOMContentLoaded', function(event) {
+document.addEventListener('DOMContentLoaded', async () => {
     const checkingElement = document.querySelector('.app-checking-availability');
     if (!checkingElement)
         return;
@@ -225,11 +225,7 @@ document.addEventListener('DOMContentLoaded', function(event) {
 
     function setInstallButtonState(enable) {
         const installButton = document.querySelector('.install-button');
-        if (enable) {
-            installButton.removeAttribute('disabled');
-        } else {
-            installButton.setAttribute('disabled', 'disabled');
-        }
+        installButton?.setAttribute('disabled', !enable);
     }
 
     function unavailable() {
@@ -241,37 +237,30 @@ document.addEventListener('DOMContentLoaded', function(event) {
         const element = document.querySelector('.app-checking-availability-error');
         element.classList.remove('d-none');
         checkingElement.classList.add('d-none');
-        setInstallButtonState(true);  // Allow trying installation
+        setInstallButtonState(true); // Allow trying installation
     }
 
-    let request = new XMLHttpRequest();
-    request.timeout = 2 * 60 * 1000;  // 2 minutes
-    request.onload = function() {
-        // Remove the progress spinner
+    try {
+        setInstallButtonState(false);
+        const response = await fetch(`/plinth/is-available/${appId}/`, {
+            timeout: 2 * 60 * 1000  // 2 minutes
+        });
+
         checkingElement.classList.add('d-none');
 
-        let available = false;
-        if (this.status === 200) {
-            try {
-                const response = JSON.parse(this.responseText);
-                if (response.is_available === true) {
-                    setInstallButtonState(true);
-                } else if (response.is_available === false) {
-                    unavailable();
-                } else {
-                    error();
-                }
-            } catch (e) {
+        if (response.ok) {
+            const data = await response.json();
+            if (data.is_available === true) {
+                setInstallButtonState(true);
+            } else if (data.is_available === false) {
+                unavailable();
+            } else {
                 error();
             }
         } else {
             error();
         }
-    };
-    request.onerror = error;
-    request.ontimeout = error;
-
-    request.open('GET', `/plinth/is-available/${appId}/`, true);
-    setInstallButtonState(false);
-    request.send();
+    } catch {
+        error();
+    }
 });