diff --git a/debian/freedombox.maintscript b/debian/freedombox.maintscript
index 3a4aaa393..d46134038 100644
--- a/debian/freedombox.maintscript
+++ b/debian/freedombox.maintscript
@@ -139,3 +139,7 @@ rm_conffile /etc/apache2/includes/freedombox-single-sign-on.conf 23.10~
 rm_conffile /etc/apache2/conf-available/syncthing-plinth.conf 23.10~
 rm_conffile /etc/apache2/conf-available/transmission-plinth.conf 23.10~
 rm_conffile /etc/apache2/conf-available/tt-rss-plinth.conf 23.10~
+rm_conffile /etc/apt/apt.conf.d/20freedombox 23.10~
+rm_conffile /etc/apt/apt.conf.d/20freedombox-allow-release-info-change 23.10~
+rm_conffile /etc/apt/apt.conf.d/60unattended-upgrades 23.10~
+rm_conffile /etc/needrestart/conf.d/freedombox.conf 23.10~
diff --git a/plinth/modules/upgrades/__init__.py b/plinth/modules/upgrades/__init__.py
index 4a732df83..72435c8a6 100644
--- a/plinth/modules/upgrades/__init__.py
+++ b/plinth/modules/upgrades/__init__.py
@@ -12,6 +12,7 @@ from django.utils.translation import gettext_noop
 import plinth
 from plinth import app as app_module
 from plinth import cfg, glib, kvstore, menu
+from plinth.config import DropinConfigs
 from plinth.daemon import RelatedDaemon
 from plinth.modules.backups.components import BackupRestore
 from plinth.package import Packages
@@ -52,7 +53,7 @@ class UpgradesApp(app_module.App):
 
     app_id = 'upgrades'
 
-    _version = 15
+    _version = 16
 
     can_be_disabled = False
 
@@ -74,6 +75,14 @@ class UpgradesApp(app_module.App):
                             ['unattended-upgrades', 'needrestart'])
         self.add(packages)
 
+        dropin_configs = DropinConfigs('dropin-configs-upgrades', [
+            '/etc/apt/apt.conf.d/20freedombox',
+            '/etc/apt/apt.conf.d/20freedombox-allow-release-info-change',
+            '/etc/apt/apt.conf.d/60unattended-upgrades',
+            '/etc/needrestart/conf.d/freedombox.conf',
+        ])
+        self.add(dropin_configs)
+
         daemon = RelatedDaemon('related-daemon-upgrades',
                                'freedombox-dist-upgrade')
         self.add(daemon)
diff --git a/plinth/modules/upgrades/data/etc/apt/apt.conf.d/20freedombox b/plinth/modules/upgrades/data/usr/share/freedombox/etc/apt/apt.conf.d/20freedombox
similarity index 100%
rename from plinth/modules/upgrades/data/etc/apt/apt.conf.d/20freedombox
rename to plinth/modules/upgrades/data/usr/share/freedombox/etc/apt/apt.conf.d/20freedombox
diff --git a/plinth/modules/upgrades/data/etc/apt/apt.conf.d/20freedombox-allow-release-info-change b/plinth/modules/upgrades/data/usr/share/freedombox/etc/apt/apt.conf.d/20freedombox-allow-release-info-change
similarity index 100%
rename from plinth/modules/upgrades/data/etc/apt/apt.conf.d/20freedombox-allow-release-info-change
rename to plinth/modules/upgrades/data/usr/share/freedombox/etc/apt/apt.conf.d/20freedombox-allow-release-info-change
diff --git a/plinth/modules/upgrades/data/etc/apt/apt.conf.d/60unattended-upgrades b/plinth/modules/upgrades/data/usr/share/freedombox/etc/apt/apt.conf.d/60unattended-upgrades
similarity index 100%
rename from plinth/modules/upgrades/data/etc/apt/apt.conf.d/60unattended-upgrades
rename to plinth/modules/upgrades/data/usr/share/freedombox/etc/apt/apt.conf.d/60unattended-upgrades
diff --git a/plinth/modules/upgrades/data/etc/needrestart/conf.d/freedombox.conf b/plinth/modules/upgrades/data/usr/share/freedombox/etc/needrestart/conf.d/freedombox.conf
similarity index 100%
rename from plinth/modules/upgrades/data/etc/needrestart/conf.d/freedombox.conf
rename to plinth/modules/upgrades/data/usr/share/freedombox/etc/needrestart/conf.d/freedombox.conf