diff --git a/debian/freedombox.maintscript b/debian/freedombox.maintscript
index ee1a3180b..835bba07c 100644
--- a/debian/freedombox.maintscript
+++ b/debian/freedombox.maintscript
@@ -144,3 +144,7 @@ rm_conffile /etc/apt/apt.conf.d/20freedombox-allow-release-info-change 23.10~
 rm_conffile /etc/apt/apt.conf.d/60unattended-upgrades 23.10~
 rm_conffile /etc/needrestart/conf.d/freedombox.conf 23.10~
 rm_conffile /etc/apache2/includes/freedombox-auth-ldap.conf 23.10~
+rm_conffile /etc/apache2/conf-available/wordpress-freedombox.conf 23.10~
+rm_conffile /etc/fail2ban/jail.d/wordpress-freedombox.conf 23.10~
+rm_conffile /etc/fail2ban/filter.d/wordpress-freedombox.conf 23.10~
+rm_conffile /etc/wordpress/freedombox-static.php 23.10~
diff --git a/plinth/modules/wordpress/__init__.py b/plinth/modules/wordpress/__init__.py
index 62571537b..f584e21b3 100644
--- a/plinth/modules/wordpress/__init__.py
+++ b/plinth/modules/wordpress/__init__.py
@@ -5,6 +5,7 @@ from django.utils.translation import gettext_lazy as _
 
 from plinth import app as app_module
 from plinth import cfg, frontpage, menu
+from plinth.config import DropinConfigs
 from plinth.daemon import Daemon
 from plinth.modules.apache.components import Webserver
 from plinth.modules.backups.components import BackupRestore
@@ -42,7 +43,7 @@ class WordPressApp(app_module.App):
 
     app_id = 'wordpress'
 
-    _version = 3
+    _version = 4
 
     def __init__(self):
         """Create components for the app."""
@@ -86,6 +87,14 @@ class WordPressApp(app_module.App):
             conflicts_action=Packages.ConflictsAction.REMOVE)
         self.add(packages)
 
+        dropin_configs = DropinConfigs('dropin-configs-wordpress', [
+            '/etc/apache2/conf-available/wordpress-freedombox.conf',
+            '/etc/fail2ban/jail.d/wordpress-freedombox.conf',
+            '/etc/fail2ban/filter.d/wordpress-freedombox.conf',
+            '/etc/wordpress/freedombox-static.php',
+        ])
+        self.add(dropin_configs)
+
         firewall = Firewall('firewall-wordpress', info.name,
                             ports=['http', 'https'], is_external=True)
         self.add(firewall)
diff --git a/plinth/modules/wordpress/data/etc/apache2/conf-available/wordpress-freedombox.conf b/plinth/modules/wordpress/data/usr/share/freedombox/etc/apache2/conf-available/wordpress-freedombox.conf
similarity index 100%
rename from plinth/modules/wordpress/data/etc/apache2/conf-available/wordpress-freedombox.conf
rename to plinth/modules/wordpress/data/usr/share/freedombox/etc/apache2/conf-available/wordpress-freedombox.conf
diff --git a/plinth/modules/wordpress/data/etc/fail2ban/filter.d/wordpress-freedombox.conf b/plinth/modules/wordpress/data/usr/share/freedombox/etc/fail2ban/filter.d/wordpress-freedombox.conf
similarity index 100%
rename from plinth/modules/wordpress/data/etc/fail2ban/filter.d/wordpress-freedombox.conf
rename to plinth/modules/wordpress/data/usr/share/freedombox/etc/fail2ban/filter.d/wordpress-freedombox.conf
diff --git a/plinth/modules/wordpress/data/etc/fail2ban/jail.d/wordpress-freedombox.conf b/plinth/modules/wordpress/data/usr/share/freedombox/etc/fail2ban/jail.d/wordpress-freedombox.conf
similarity index 100%
rename from plinth/modules/wordpress/data/etc/fail2ban/jail.d/wordpress-freedombox.conf
rename to plinth/modules/wordpress/data/usr/share/freedombox/etc/fail2ban/jail.d/wordpress-freedombox.conf
diff --git a/plinth/modules/wordpress/data/etc/wordpress/freedombox-static.php b/plinth/modules/wordpress/data/usr/share/freedombox/etc/wordpress/freedombox-static.php
similarity index 100%
rename from plinth/modules/wordpress/data/etc/wordpress/freedombox-static.php
rename to plinth/modules/wordpress/data/usr/share/freedombox/etc/wordpress/freedombox-static.php