diff --git a/plinth/kvstore.py b/plinth/kvstore.py
index 48911387e..477ecc195 100644
--- a/plinth/kvstore.py
+++ b/plinth/kvstore.py
@@ -27,7 +27,11 @@ def set(key, value):  # pylint: disable-msg=W0622
     store.save()
 
 
-def delete(key):
+def delete(key, ignore_missing=False):
     """Delete a key"""
     from plinth.models import KVStore
-    return KVStore.objects.get(key=key).delete()
+    try:
+        return KVStore.objects.get(key=key).delete()
+    except KVStore.DoesNotExist:
+        if not ignore_missing:
+            raise
diff --git a/plinth/tests/test_kvstore.py b/plinth/tests/test_kvstore.py
index b4e3d687e..e29b71c73 100644
--- a/plinth/tests/test_kvstore.py
+++ b/plinth/tests/test_kvstore.py
@@ -6,6 +6,7 @@ Test module for key/value store.
 import pytest
 
 from plinth import kvstore
+from plinth.models import KVStore
 
 pytestmark = pytest.mark.django_db
 
@@ -41,3 +42,19 @@ def test_get_default():
     expected = 'default'
     actual = kvstore.get_default('bad_key', expected)
     assert expected == actual
+
+
+def test_delete():
+    """Test that deleting key works."""
+    with pytest.raises(KVStore.DoesNotExist):
+        kvstore.delete('nonexistant_key')
+
+    with pytest.raises(KVStore.DoesNotExist):
+        kvstore.delete('nonexistant_key', ignore_missing=False)
+
+    kvstore.delete('nonexistant_key', ignore_missing=True)
+
+    kvstore.set('test-set-key', 'test-value')
+    kvstore.delete('test-set-key')
+    with pytest.raises(KVStore.DoesNotExist):
+        kvstore.delete('test-set-key')