diff --git a/debian/freedombox.maintscript b/debian/freedombox.maintscript
index 0784752f1..94aaf81f5 100644
--- a/debian/freedombox.maintscript
+++ b/debian/freedombox.maintscript
@@ -129,3 +129,5 @@ rm_conffile /etc/fail2ban/jail.d/roundcube-auth-freedombox.conf 23.10~
 rm_conffile /etc/apache2/conf-available/rss-bridge.conf 23.10~
 rm_conffile /etc/apache2/conf-available/searx-freedombox-auth.conf 23.10~
 rm_conffile /etc/apache2/conf-available/searx-freedombox.conf 23.10~
+rm_conffile /etc/fail2ban/fail2ban.d/freedombox.conf 23.10~
+rm_conffile /etc/fail2ban/jail.d/freedombox.conf 23.10~
diff --git a/plinth/modules/security/__init__.py b/plinth/modules/security/__init__.py
index 012e85009..ecaf38555 100644
--- a/plinth/modules/security/__init__.py
+++ b/plinth/modules/security/__init__.py
@@ -9,6 +9,7 @@ from django.utils.translation import gettext_lazy as _
 
 from plinth import app as app_module
 from plinth import menu
+from plinth.config import DropinConfigs
 from plinth.daemon import Daemon, RelatedDaemon
 from plinth.modules.backups.components import BackupRestore
 from plinth.package import Packages
@@ -22,7 +23,7 @@ class SecurityApp(app_module.App):
 
     app_id = 'security'
 
-    _version = 8
+    _version = 9
 
     can_be_disabled = False
 
@@ -42,6 +43,12 @@ class SecurityApp(app_module.App):
         packages = Packages('packages-security', ['fail2ban', 'debsecan'])
         self.add(packages)
 
+        dropin_configs = DropinConfigs('dropin-configs-security', [
+            '/etc/fail2ban/fail2ban.d/freedombox.conf',
+            '/etc/fail2ban/jail.d/freedombox.conf',
+        ])
+        self.add(dropin_configs)
+
         daemon = RelatedDaemon('related-daemon-fail2ban', 'fail2ban')
         self.add(daemon)
 
diff --git a/plinth/modules/security/data/etc/fail2ban/fail2ban.d/freedombox.conf b/plinth/modules/security/data/usr/share/freedombox/etc/fail2ban/fail2ban.d/freedombox.conf
similarity index 100%
rename from plinth/modules/security/data/etc/fail2ban/fail2ban.d/freedombox.conf
rename to plinth/modules/security/data/usr/share/freedombox/etc/fail2ban/fail2ban.d/freedombox.conf
diff --git a/plinth/modules/security/data/etc/fail2ban/jail.d/freedombox.conf b/plinth/modules/security/data/usr/share/freedombox/etc/fail2ban/jail.d/freedombox.conf
similarity index 100%
rename from plinth/modules/security/data/etc/fail2ban/jail.d/freedombox.conf
rename to plinth/modules/security/data/usr/share/freedombox/etc/fail2ban/jail.d/freedombox.conf