Deluan Quintão 2471bb9cf6
feat(plugins): add TTL support, batch operations, and hardening to kvstore (#5127)
* feat(plugins): add expires_at column to kvstore schema

* feat(plugins): filter expired keys in kvstore Get, Has, List

* feat(plugins): add periodic cleanup of expired kvstore keys

* feat(plugins): add SetWithTTL, DeleteByPrefix, and GetMany to kvstore

Add three new methods to the KVStore host service:

- SetWithTTL: store key-value pairs with automatic expiration
- DeleteByPrefix: remove all keys matching a prefix in one operation
- GetMany: retrieve multiple values in a single call

All methods include comprehensive unit tests covering edge cases,
expiration behavior, size tracking, and LIKE-special characters.

* feat(plugins): regenerate code and update test plugin for new kvstore methods

Regenerate host function wrappers and PDK bindings for Go, Python,
and Rust. Update the test-kvstore plugin to exercise SetWithTTL,
DeleteByPrefix, and GetMany.

* feat(plugins): add integration tests for new kvstore methods

Add WASM integration tests for SetWithTTL, DeleteByPrefix, and GetMany
operations through the plugin boundary, verifying end-to-end behavior
including TTL expiration, prefix deletion, and batch retrieval.

* fix(plugins): address lint issues in kvstore implementation

Handle tx.Rollback error return and suppress gosec false positive
for parameterized SQL query construction in GetMany.

* fix(plugins): Set clears expires_at when overwriting a TTL'd key

Previously, calling Set() on a key that was stored with SetWithTTL()
would leave the expires_at value intact, causing the key to silently
expire even though Set implies permanent storage.

Also excludes expired keys from currentSize calculation at startup.

* refactor(plugins): simplify kvstore by removing in-memory size cache

Replaced the in-memory currentSize cache (atomic.Int64), periodic cleanup
timer, and mutex with direct database queries for storage accounting.
This eliminates race conditions and cache drift issues at negligible
performance cost for plugin-sized datasets. Also unified Set and
SetWithTTL into a shared setValue method, simplified DeleteByPrefix to
use RowsAffected instead of a transaction, and added an index on
expires_at for efficient expiration filtering.

* feat(plugins): add generic SQLite migration helper and refactor kvstore schema

Add a reusable migrateDB helper that tracks schema versions via SQLite's
PRAGMA user_version and applies pending migrations transactionally. Replace
the ad-hoc createKVStoreSchema function in kvstore with a declarative
migrations slice, making it easy to add future schema changes. Remove the
now-redundant schema migration test since migrateDB has its own test suite
and every kvstore test exercises the migrations implicitly.

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(plugins): harden kvstore with explicit NULL handling, prefix validation, and cleanup timeout

- Use sql.NullString for expires_at to explicitly send NULL instead of
  relying on datetime('now', '') returning NULL by accident
- Reject empty prefix in DeleteByPrefix to prevent accidental data wipe
- Add 5s timeout context to cleanupExpired on Close
- Replace time.Sleep in unit tests with pre-expired timestamps

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(plugins): use batch processing in GetMany

Process keys in chunks of 200 using slice.CollectChunks to avoid
hitting SQLite's SQLITE_MAX_VARIABLE_NUMBER limit with large key sets.

* feat(plugins): add periodic cleanup goroutine for expired kvstore keys

Use the manager's context to control a background goroutine that purges
expired keys every hour, stopping naturally on shutdown when the context
is cancelled.

---------

Signed-off-by: Deluan <deluan@navidrome.org>
2026-02-28 23:12:17 -05:00

141 lines
4.0 KiB
Go

// Test KVStore plugin for Navidrome plugin system integration tests.
// Build with: tinygo build -o ../test-kvstore.wasm -target wasip1 -buildmode=c-shared .
package main
import (
"github.com/navidrome/navidrome/plugins/pdk/go/host"
"github.com/navidrome/navidrome/plugins/pdk/go/pdk"
)
// TestKVStoreInput is the input for nd_test_kvstore callback.
type TestKVStoreInput struct {
Operation string `json:"operation"` // "set", "get", "delete", "has", "list", "get_storage_used", "set_with_ttl", "delete_by_prefix", "get_many"
Key string `json:"key"` // Storage key
Value []byte `json:"value"` // For set operations
Prefix string `json:"prefix"` // For list/delete_by_prefix operations
TTLSeconds int64 `json:"ttl_seconds,omitempty"` // For set_with_ttl
Keys []string `json:"keys,omitempty"` // For get_many
}
// TestKVStoreOutput is the output from nd_test_kvstore callback.
type TestKVStoreOutput struct {
Value []byte `json:"value,omitempty"`
Values map[string][]byte `json:"values,omitempty"`
Exists bool `json:"exists,omitempty"`
Keys []string `json:"keys,omitempty"`
StorageUsed int64 `json:"storage_used,omitempty"`
DeletedCount int64 `json:"deleted_count,omitempty"`
Error *string `json:"error,omitempty"`
}
// nd_test_kvstore is the test callback that tests the kvstore host functions.
//
//go:wasmexport nd_test_kvstore
func ndTestKVStore() int32 {
var input TestKVStoreInput
if err := pdk.InputJSON(&input); err != nil {
errStr := err.Error()
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
return 0
}
switch input.Operation {
case "set":
err := host.KVStoreSet(input.Key, input.Value)
if err != nil {
errStr := err.Error()
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
return 0
}
pdk.OutputJSON(TestKVStoreOutput{})
return 0
case "get":
value, exists, err := host.KVStoreGet(input.Key)
if err != nil {
errStr := err.Error()
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
return 0
}
pdk.OutputJSON(TestKVStoreOutput{Value: value, Exists: exists})
return 0
case "delete":
err := host.KVStoreDelete(input.Key)
if err != nil {
errStr := err.Error()
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
return 0
}
pdk.OutputJSON(TestKVStoreOutput{})
return 0
case "has":
exists, err := host.KVStoreHas(input.Key)
if err != nil {
errStr := err.Error()
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
return 0
}
pdk.OutputJSON(TestKVStoreOutput{Exists: exists})
return 0
case "list":
keys, err := host.KVStoreList(input.Prefix)
if err != nil {
errStr := err.Error()
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
return 0
}
pdk.OutputJSON(TestKVStoreOutput{Keys: keys})
return 0
case "get_storage_used":
bytesUsed, err := host.KVStoreGetStorageUsed()
if err != nil {
errStr := err.Error()
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
return 0
}
pdk.OutputJSON(TestKVStoreOutput{StorageUsed: bytesUsed})
return 0
case "set_with_ttl":
err := host.KVStoreSetWithTTL(input.Key, input.Value, input.TTLSeconds)
if err != nil {
errStr := err.Error()
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
return 0
}
pdk.OutputJSON(TestKVStoreOutput{})
return 0
case "delete_by_prefix":
deletedCount, err := host.KVStoreDeleteByPrefix(input.Prefix)
if err != nil {
errStr := err.Error()
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
return 0
}
pdk.OutputJSON(TestKVStoreOutput{DeletedCount: deletedCount})
return 0
case "get_many":
values, err := host.KVStoreGetMany(input.Keys)
if err != nil {
errStr := err.Error()
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
return 0
}
pdk.OutputJSON(TestKVStoreOutput{Values: values})
return 0
default:
errStr := "unknown operation: " + input.Operation
pdk.OutputJSON(TestKVStoreOutput{Error: &errStr})
return 0
}
}
func main() {}