mirror of
https://github.com/navidrome/navidrome.git
synced 2026-04-03 06:41:01 +00:00
* feat(plugins): add expires_at column to kvstore schema
* feat(plugins): filter expired keys in kvstore Get, Has, List
* feat(plugins): add periodic cleanup of expired kvstore keys
* feat(plugins): add SetWithTTL, DeleteByPrefix, and GetMany to kvstore
Add three new methods to the KVStore host service:
- SetWithTTL: store key-value pairs with automatic expiration
- DeleteByPrefix: remove all keys matching a prefix in one operation
- GetMany: retrieve multiple values in a single call
All methods include comprehensive unit tests covering edge cases,
expiration behavior, size tracking, and LIKE-special characters.
* feat(plugins): regenerate code and update test plugin for new kvstore methods
Regenerate host function wrappers and PDK bindings for Go, Python,
and Rust. Update the test-kvstore plugin to exercise SetWithTTL,
DeleteByPrefix, and GetMany.
* feat(plugins): add integration tests for new kvstore methods
Add WASM integration tests for SetWithTTL, DeleteByPrefix, and GetMany
operations through the plugin boundary, verifying end-to-end behavior
including TTL expiration, prefix deletion, and batch retrieval.
* fix(plugins): address lint issues in kvstore implementation
Handle tx.Rollback error return and suppress gosec false positive
for parameterized SQL query construction in GetMany.
* fix(plugins): Set clears expires_at when overwriting a TTL'd key
Previously, calling Set() on a key that was stored with SetWithTTL()
would leave the expires_at value intact, causing the key to silently
expire even though Set implies permanent storage.
Also excludes expired keys from currentSize calculation at startup.
* refactor(plugins): simplify kvstore by removing in-memory size cache
Replaced the in-memory currentSize cache (atomic.Int64), periodic cleanup
timer, and mutex with direct database queries for storage accounting.
This eliminates race conditions and cache drift issues at negligible
performance cost for plugin-sized datasets. Also unified Set and
SetWithTTL into a shared setValue method, simplified DeleteByPrefix to
use RowsAffected instead of a transaction, and added an index on
expires_at for efficient expiration filtering.
* feat(plugins): add generic SQLite migration helper and refactor kvstore schema
Add a reusable migrateDB helper that tracks schema versions via SQLite's
PRAGMA user_version and applies pending migrations transactionally. Replace
the ad-hoc createKVStoreSchema function in kvstore with a declarative
migrations slice, making it easy to add future schema changes. Remove the
now-redundant schema migration test since migrateDB has its own test suite
and every kvstore test exercises the migrations implicitly.
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(plugins): harden kvstore with explicit NULL handling, prefix validation, and cleanup timeout
- Use sql.NullString for expires_at to explicitly send NULL instead of
relying on datetime('now', '') returning NULL by accident
- Reject empty prefix in DeleteByPrefix to prevent accidental data wipe
- Add 5s timeout context to cleanupExpired on Close
- Replace time.Sleep in unit tests with pre-expired timestamps
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor(plugins): use batch processing in GetMany
Process keys in chunks of 200 using slice.CollectChunks to avoid
hitting SQLite's SQLITE_MAX_VARIABLE_NUMBER limit with large key sets.
* feat(plugins): add periodic cleanup goroutine for expired kvstore keys
Use the manager's context to control a background goroutine that purges
expired keys every hour, stopping naturally on shutdown when the context
is cancelled.
---------
Signed-off-by: Deluan <deluan@navidrome.org>
434 lines
12 KiB
Go
434 lines
12 KiB
Go
// Code generated by ndpgen. DO NOT EDIT.
|
|
|
|
package host
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
|
|
extism "github.com/extism/go-sdk"
|
|
)
|
|
|
|
// KVStoreSetRequest is the request type for KVStore.Set.
|
|
type KVStoreSetRequest struct {
|
|
Key string `json:"key"`
|
|
Value []byte `json:"value"`
|
|
}
|
|
|
|
// KVStoreSetResponse is the response type for KVStore.Set.
|
|
type KVStoreSetResponse struct {
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// KVStoreSetWithTTLRequest is the request type for KVStore.SetWithTTL.
|
|
type KVStoreSetWithTTLRequest struct {
|
|
Key string `json:"key"`
|
|
Value []byte `json:"value"`
|
|
TtlSeconds int64 `json:"ttlSeconds"`
|
|
}
|
|
|
|
// KVStoreSetWithTTLResponse is the response type for KVStore.SetWithTTL.
|
|
type KVStoreSetWithTTLResponse struct {
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// KVStoreGetRequest is the request type for KVStore.Get.
|
|
type KVStoreGetRequest struct {
|
|
Key string `json:"key"`
|
|
}
|
|
|
|
// KVStoreGetResponse is the response type for KVStore.Get.
|
|
type KVStoreGetResponse struct {
|
|
Value []byte `json:"value,omitempty"`
|
|
Exists bool `json:"exists,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// KVStoreGetManyRequest is the request type for KVStore.GetMany.
|
|
type KVStoreGetManyRequest struct {
|
|
Keys []string `json:"keys"`
|
|
}
|
|
|
|
// KVStoreGetManyResponse is the response type for KVStore.GetMany.
|
|
type KVStoreGetManyResponse struct {
|
|
Values map[string][]byte `json:"values,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// KVStoreHasRequest is the request type for KVStore.Has.
|
|
type KVStoreHasRequest struct {
|
|
Key string `json:"key"`
|
|
}
|
|
|
|
// KVStoreHasResponse is the response type for KVStore.Has.
|
|
type KVStoreHasResponse struct {
|
|
Exists bool `json:"exists,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// KVStoreListRequest is the request type for KVStore.List.
|
|
type KVStoreListRequest struct {
|
|
Prefix string `json:"prefix"`
|
|
}
|
|
|
|
// KVStoreListResponse is the response type for KVStore.List.
|
|
type KVStoreListResponse struct {
|
|
Keys []string `json:"keys,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// KVStoreDeleteRequest is the request type for KVStore.Delete.
|
|
type KVStoreDeleteRequest struct {
|
|
Key string `json:"key"`
|
|
}
|
|
|
|
// KVStoreDeleteResponse is the response type for KVStore.Delete.
|
|
type KVStoreDeleteResponse struct {
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// KVStoreDeleteByPrefixRequest is the request type for KVStore.DeleteByPrefix.
|
|
type KVStoreDeleteByPrefixRequest struct {
|
|
Prefix string `json:"prefix"`
|
|
}
|
|
|
|
// KVStoreDeleteByPrefixResponse is the response type for KVStore.DeleteByPrefix.
|
|
type KVStoreDeleteByPrefixResponse struct {
|
|
DeletedCount int64 `json:"deletedCount,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// KVStoreGetStorageUsedResponse is the response type for KVStore.GetStorageUsed.
|
|
type KVStoreGetStorageUsedResponse struct {
|
|
Bytes int64 `json:"bytes,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// RegisterKVStoreHostFunctions registers KVStore service host functions.
|
|
// The returned host functions should be added to the plugin's configuration.
|
|
func RegisterKVStoreHostFunctions(service KVStoreService) []extism.HostFunction {
|
|
return []extism.HostFunction{
|
|
newKVStoreSetHostFunction(service),
|
|
newKVStoreSetWithTTLHostFunction(service),
|
|
newKVStoreGetHostFunction(service),
|
|
newKVStoreGetManyHostFunction(service),
|
|
newKVStoreHasHostFunction(service),
|
|
newKVStoreListHostFunction(service),
|
|
newKVStoreDeleteHostFunction(service),
|
|
newKVStoreDeleteByPrefixHostFunction(service),
|
|
newKVStoreGetStorageUsedHostFunction(service),
|
|
}
|
|
}
|
|
|
|
func newKVStoreSetHostFunction(service KVStoreService) extism.HostFunction {
|
|
return extism.NewHostFunctionWithStack(
|
|
"kvstore_set",
|
|
func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
|
|
// Read JSON request from plugin memory
|
|
reqBytes, err := p.ReadBytes(stack[0])
|
|
if err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
var req KVStoreSetRequest
|
|
if err := json.Unmarshal(reqBytes, &req); err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
|
|
// Call the service method
|
|
if svcErr := service.Set(ctx, req.Key, req.Value); svcErr != nil {
|
|
kvstoreWriteError(p, stack, svcErr)
|
|
return
|
|
}
|
|
|
|
// Write JSON response to plugin memory
|
|
resp := KVStoreSetResponse{}
|
|
kvstoreWriteResponse(p, stack, resp)
|
|
},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
)
|
|
}
|
|
|
|
func newKVStoreSetWithTTLHostFunction(service KVStoreService) extism.HostFunction {
|
|
return extism.NewHostFunctionWithStack(
|
|
"kvstore_setwithttl",
|
|
func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
|
|
// Read JSON request from plugin memory
|
|
reqBytes, err := p.ReadBytes(stack[0])
|
|
if err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
var req KVStoreSetWithTTLRequest
|
|
if err := json.Unmarshal(reqBytes, &req); err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
|
|
// Call the service method
|
|
if svcErr := service.SetWithTTL(ctx, req.Key, req.Value, req.TtlSeconds); svcErr != nil {
|
|
kvstoreWriteError(p, stack, svcErr)
|
|
return
|
|
}
|
|
|
|
// Write JSON response to plugin memory
|
|
resp := KVStoreSetWithTTLResponse{}
|
|
kvstoreWriteResponse(p, stack, resp)
|
|
},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
)
|
|
}
|
|
|
|
func newKVStoreGetHostFunction(service KVStoreService) extism.HostFunction {
|
|
return extism.NewHostFunctionWithStack(
|
|
"kvstore_get",
|
|
func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
|
|
// Read JSON request from plugin memory
|
|
reqBytes, err := p.ReadBytes(stack[0])
|
|
if err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
var req KVStoreGetRequest
|
|
if err := json.Unmarshal(reqBytes, &req); err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
|
|
// Call the service method
|
|
value, exists, svcErr := service.Get(ctx, req.Key)
|
|
if svcErr != nil {
|
|
kvstoreWriteError(p, stack, svcErr)
|
|
return
|
|
}
|
|
|
|
// Write JSON response to plugin memory
|
|
resp := KVStoreGetResponse{
|
|
Value: value,
|
|
Exists: exists,
|
|
}
|
|
kvstoreWriteResponse(p, stack, resp)
|
|
},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
)
|
|
}
|
|
|
|
func newKVStoreGetManyHostFunction(service KVStoreService) extism.HostFunction {
|
|
return extism.NewHostFunctionWithStack(
|
|
"kvstore_getmany",
|
|
func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
|
|
// Read JSON request from plugin memory
|
|
reqBytes, err := p.ReadBytes(stack[0])
|
|
if err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
var req KVStoreGetManyRequest
|
|
if err := json.Unmarshal(reqBytes, &req); err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
|
|
// Call the service method
|
|
values, svcErr := service.GetMany(ctx, req.Keys)
|
|
if svcErr != nil {
|
|
kvstoreWriteError(p, stack, svcErr)
|
|
return
|
|
}
|
|
|
|
// Write JSON response to plugin memory
|
|
resp := KVStoreGetManyResponse{
|
|
Values: values,
|
|
}
|
|
kvstoreWriteResponse(p, stack, resp)
|
|
},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
)
|
|
}
|
|
|
|
func newKVStoreHasHostFunction(service KVStoreService) extism.HostFunction {
|
|
return extism.NewHostFunctionWithStack(
|
|
"kvstore_has",
|
|
func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
|
|
// Read JSON request from plugin memory
|
|
reqBytes, err := p.ReadBytes(stack[0])
|
|
if err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
var req KVStoreHasRequest
|
|
if err := json.Unmarshal(reqBytes, &req); err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
|
|
// Call the service method
|
|
exists, svcErr := service.Has(ctx, req.Key)
|
|
if svcErr != nil {
|
|
kvstoreWriteError(p, stack, svcErr)
|
|
return
|
|
}
|
|
|
|
// Write JSON response to plugin memory
|
|
resp := KVStoreHasResponse{
|
|
Exists: exists,
|
|
}
|
|
kvstoreWriteResponse(p, stack, resp)
|
|
},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
)
|
|
}
|
|
|
|
func newKVStoreListHostFunction(service KVStoreService) extism.HostFunction {
|
|
return extism.NewHostFunctionWithStack(
|
|
"kvstore_list",
|
|
func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
|
|
// Read JSON request from plugin memory
|
|
reqBytes, err := p.ReadBytes(stack[0])
|
|
if err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
var req KVStoreListRequest
|
|
if err := json.Unmarshal(reqBytes, &req); err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
|
|
// Call the service method
|
|
keys, svcErr := service.List(ctx, req.Prefix)
|
|
if svcErr != nil {
|
|
kvstoreWriteError(p, stack, svcErr)
|
|
return
|
|
}
|
|
|
|
// Write JSON response to plugin memory
|
|
resp := KVStoreListResponse{
|
|
Keys: keys,
|
|
}
|
|
kvstoreWriteResponse(p, stack, resp)
|
|
},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
)
|
|
}
|
|
|
|
func newKVStoreDeleteHostFunction(service KVStoreService) extism.HostFunction {
|
|
return extism.NewHostFunctionWithStack(
|
|
"kvstore_delete",
|
|
func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
|
|
// Read JSON request from plugin memory
|
|
reqBytes, err := p.ReadBytes(stack[0])
|
|
if err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
var req KVStoreDeleteRequest
|
|
if err := json.Unmarshal(reqBytes, &req); err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
|
|
// Call the service method
|
|
if svcErr := service.Delete(ctx, req.Key); svcErr != nil {
|
|
kvstoreWriteError(p, stack, svcErr)
|
|
return
|
|
}
|
|
|
|
// Write JSON response to plugin memory
|
|
resp := KVStoreDeleteResponse{}
|
|
kvstoreWriteResponse(p, stack, resp)
|
|
},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
)
|
|
}
|
|
|
|
func newKVStoreDeleteByPrefixHostFunction(service KVStoreService) extism.HostFunction {
|
|
return extism.NewHostFunctionWithStack(
|
|
"kvstore_deletebyprefix",
|
|
func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
|
|
// Read JSON request from plugin memory
|
|
reqBytes, err := p.ReadBytes(stack[0])
|
|
if err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
var req KVStoreDeleteByPrefixRequest
|
|
if err := json.Unmarshal(reqBytes, &req); err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
|
|
// Call the service method
|
|
deletedcount, svcErr := service.DeleteByPrefix(ctx, req.Prefix)
|
|
if svcErr != nil {
|
|
kvstoreWriteError(p, stack, svcErr)
|
|
return
|
|
}
|
|
|
|
// Write JSON response to plugin memory
|
|
resp := KVStoreDeleteByPrefixResponse{
|
|
DeletedCount: deletedcount,
|
|
}
|
|
kvstoreWriteResponse(p, stack, resp)
|
|
},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
)
|
|
}
|
|
|
|
func newKVStoreGetStorageUsedHostFunction(service KVStoreService) extism.HostFunction {
|
|
return extism.NewHostFunctionWithStack(
|
|
"kvstore_getstorageused",
|
|
func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
|
|
|
|
// Call the service method
|
|
bytes, svcErr := service.GetStorageUsed(ctx)
|
|
if svcErr != nil {
|
|
kvstoreWriteError(p, stack, svcErr)
|
|
return
|
|
}
|
|
|
|
// Write JSON response to plugin memory
|
|
resp := KVStoreGetStorageUsedResponse{
|
|
Bytes: bytes,
|
|
}
|
|
kvstoreWriteResponse(p, stack, resp)
|
|
},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
[]extism.ValueType{extism.ValueTypePTR},
|
|
)
|
|
}
|
|
|
|
// kvstoreWriteResponse writes a JSON response to plugin memory.
|
|
func kvstoreWriteResponse(p *extism.CurrentPlugin, stack []uint64, resp any) {
|
|
respBytes, err := json.Marshal(resp)
|
|
if err != nil {
|
|
kvstoreWriteError(p, stack, err)
|
|
return
|
|
}
|
|
respPtr, err := p.WriteBytes(respBytes)
|
|
if err != nil {
|
|
stack[0] = 0
|
|
return
|
|
}
|
|
stack[0] = respPtr
|
|
}
|
|
|
|
// kvstoreWriteError writes an error response to plugin memory.
|
|
func kvstoreWriteError(p *extism.CurrentPlugin, stack []uint64, err error) {
|
|
errResp := struct {
|
|
Error string `json:"error"`
|
|
}{Error: err.Error()}
|
|
respBytes, _ := json.Marshal(errResp)
|
|
respPtr, _ := p.WriteBytes(respBytes)
|
|
stack[0] = respPtr
|
|
}
|