diff --git a/debian/freedombox.maintscript b/debian/freedombox.maintscript
index d46134038..ee1a3180b 100644
--- a/debian/freedombox.maintscript
+++ b/debian/freedombox.maintscript
@@ -143,3 +143,4 @@ rm_conffile /etc/apt/apt.conf.d/20freedombox 23.10~
 rm_conffile /etc/apt/apt.conf.d/20freedombox-allow-release-info-change 23.10~
 rm_conffile /etc/apt/apt.conf.d/60unattended-upgrades 23.10~
 rm_conffile /etc/needrestart/conf.d/freedombox.conf 23.10~
+rm_conffile /etc/apache2/includes/freedombox-auth-ldap.conf 23.10~
diff --git a/plinth/modules/users/__init__.py b/plinth/modules/users/__init__.py
index 5e7f68e89..6dc4a186d 100644
--- a/plinth/modules/users/__init__.py
+++ b/plinth/modules/users/__init__.py
@@ -9,6 +9,7 @@ from django.utils.translation import gettext_lazy as _
 
 from plinth import app as app_module
 from plinth import cfg, menu
+from plinth.config import DropinConfigs
 from plinth.daemon import Daemon
 from plinth.package import Packages
 from plinth.privileged import service as service_privileged
@@ -42,7 +43,7 @@ class UsersApp(app_module.App):
 
     app_id = 'users'
 
-    _version = 4
+    _version = 5
 
     can_be_disabled = False
 
@@ -66,6 +67,11 @@ class UsersApp(app_module.App):
         ])
         self.add(packages)
 
+        dropin_configs = DropinConfigs('dropin-configs-users', [
+            '/etc/apache2/includes/freedombox-auth-ldap.conf',
+        ])
+        self.add(dropin_configs)
+
         daemon = Daemon('daemon-users', 'slapd', listen_ports=[(389, 'tcp4'),
                                                                (389, 'tcp6')])
         self.add(daemon)
diff --git a/plinth/modules/users/data/etc/apache2/includes/freedombox-auth-ldap.conf b/plinth/modules/users/data/usr/share/freedombox/etc/apache2/includes/freedombox-auth-ldap.conf
similarity index 100%
rename from plinth/modules/users/data/etc/apache2/includes/freedombox-auth-ldap.conf
rename to plinth/modules/users/data/usr/share/freedombox/etc/apache2/includes/freedombox-auth-ldap.conf