diff --git a/debian/freedombox.maintscript b/debian/freedombox.maintscript
index 835bba07c..00ce01c54 100644
--- a/debian/freedombox.maintscript
+++ b/debian/freedombox.maintscript
@@ -87,6 +87,8 @@ rm_conffile /etc/plinth/modules-enabled/users 23.10~
 rm_conffile /etc/plinth/modules-enabled/wireguard 23.10~
 rm_conffile /etc/plinth/modules-enabled/wordpress 23.10~
 rm_conffile /etc/plinth/modules-enabled/zoph 23.10~
+rm_conffile /etc/apache2/conf-available/php-fpm-freedombox.conf 23.10~
+rm_conffile /etc/fail2ban/jail.d/apache-auth-freedombox.conf 23.10~
 rm_conffile /etc/apache2/conf-available/deluge-plinth.conf 23.10~
 rm_conffile /etc/dovecot/conf.d/15-freedombox-mail.conf 23.10~
 rm_conffile /etc/dovecot/conf.d/05-freedombox-passdb.conf 23.10~
diff --git a/plinth/modules/apache/__init__.py b/plinth/modules/apache/__init__.py
index c0dffcae3..d5ead948b 100644
--- a/plinth/modules/apache/__init__.py
+++ b/plinth/modules/apache/__init__.py
@@ -7,6 +7,7 @@ from django.utils.translation import gettext_lazy as _
 
 from plinth import app as app_module
 from plinth import cfg
+from plinth.config import DropinConfigs
 from plinth.daemon import Daemon, RelatedDaemon
 from plinth.modules.firewall.components import Firewall
 from plinth.modules.letsencrypt.components import LetsEncrypt
@@ -21,7 +22,7 @@ class ApacheApp(app_module.App):
 
     app_id = 'apache'
 
-    _version = 11
+    _version = 12
 
     def __init__(self):
         """Create components for the app."""
@@ -36,6 +37,12 @@ class ApacheApp(app_module.App):
         ])
         self.add(packages)
 
+        dropin_configs = DropinConfigs('dropin-configs-apache', [
+            '/etc/apache2/conf-available/php-fpm-freedombox.conf',
+            '/etc/fail2ban/jail.d/apache-auth-freedombox.conf',
+        ])
+        self.add(dropin_configs)
+
         web_server_ports = Firewall('firewall-web', _('Web Server'),
                                     ports=['http', 'https'], is_external=True)
         self.add(web_server_ports)
diff --git a/plinth/modules/apache/data/etc/apache2/conf-available/php-fpm-freedombox.conf b/plinth/modules/apache/data/usr/share/freedombox/etc/apache2/conf-available/php-fpm-freedombox.conf
similarity index 100%
rename from plinth/modules/apache/data/etc/apache2/conf-available/php-fpm-freedombox.conf
rename to plinth/modules/apache/data/usr/share/freedombox/etc/apache2/conf-available/php-fpm-freedombox.conf
diff --git a/plinth/modules/apache/data/etc/fail2ban/jail.d/apache-auth-freedombox.conf b/plinth/modules/apache/data/usr/share/freedombox/etc/fail2ban/jail.d/apache-auth-freedombox.conf
similarity index 100%
rename from plinth/modules/apache/data/etc/fail2ban/jail.d/apache-auth-freedombox.conf
rename to plinth/modules/apache/data/usr/share/freedombox/etc/fail2ban/jail.d/apache-auth-freedombox.conf