From 6e7941b1c1d91b4bbf9471fc3fa6b8b061657761 Mon Sep 17 00:00:00 2001 From: Bernhard B Date: Sat, 23 May 2026 13:18:37 +0200 Subject: [PATCH] improved plugin mechanism + added sqlite3 plugin --- plugins/README.md | 55 ++++---- plugins/example.def | 1 + plugins/example.lua | 42 +++--- plugins/migrate-v1-plugin-to-v2.md | 29 +++++ plugins/persistence/README.md | 35 +++++ plugins/persistence/persist-message.def | 3 + plugins/persistence/persist-message.lua | 37 ++++++ plugins/persistence/query-message.def | 3 + plugins/persistence/query-message.lua | 35 +++++ src/go.mod | 7 +- src/main.go | 12 ++ src/plugin_loader.go | 163 +++++++++++++++++++++--- src/utils/plugin_config.go | 5 +- src/utils/plugin_handler.go | 1 + 14 files changed, 365 insertions(+), 63 deletions(-) create mode 100644 plugins/migrate-v1-plugin-to-v2.md create mode 100644 plugins/persistence/README.md create mode 100644 plugins/persistence/persist-message.def create mode 100644 plugins/persistence/persist-message.lua create mode 100644 plugins/persistence/query-message.def create mode 100644 plugins/persistence/query-message.lua diff --git a/plugins/README.md b/plugins/README.md index 6d7d7a2..17e76f3 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -40,45 +40,56 @@ The definition file (with the file suffix `.def`) contains some metadata which i ``` endpoint: my-custom-send-endpoint/:number method: POST +version: 2 ``` -The `endpoint` specifies the URI of the newly created endpoint. All custom endpoints are registered under the `/v1/plugins` endpoint. So, our `my-custom-send-endpoint` will be available at `/v1/plugins/my-custom-endpoint`. If you want to use variables inside the endpoint, prefix them with a `:`. +The `endpoint` specifies the URI of the newly created endpoint. All custom endpoints are registered under the `/v1/plugins` endpoint. So, our `my-custom-send-endpoint` will be available at `/v1/plugins/my-custom-endpoint`. If you want to use variables inside the endpoint, prefix them with a `:`. + +If you write a new plugin, it is recommended to use the `version: 2` of the plugin mechanism. (`version: 1` is deprecated!) The `method` parameter specifies the HTTP method that is used for the endpoint registration. # The script file -The script file (with the file suffix `.lua`) contains the implementation of the endpoint. +The script file (with the file suffix `.lua`) contains the implementation of the endpoint. Each plugin must implement a `exec` function and can optionally implement a `init` function. The `exec` function gets called whenever the plugin is called. The `init` function only gets called once during the startup and can be used to perform initialization tasks. Example: ``` -local http = require("http") -local json = require("json") +-- +function exec() + local http = require("http") + local json = require("json") -local url = "http://127.0.0.1:8080/v2/send" + local url = "http://127.0.0.1:8080/v2/send" -local customEndpointPayload = json.decode(pluginInputData.payload) + local customEndpointPayload = json.decode(pluginInputData.payload) -local sendEndpointPayload = { - recipients = {customEndpointPayload.recipient}, - message = customEndpointPayload.message, - number = pluginInputData.Params.number -} + local sendEndpointPayload = { + recipients = {customEndpointPayload.recipient}, + message = customEndpointPayload.message, + number = pluginInputData.Params.number + } -local encodedSendEndpointPayload = json.encode(sendEndpointPayload) + local encodedSendEndpointPayload = json.encode(sendEndpointPayload) -response, error_message = http.request("POST", url, { - timeout="30s", - headers={ - Accept="*/*", - ["Content-Type"]="application/json" - }, - body=encodedSendEndpointPayload -}) + response, error_message = http.request("POST", url, { + timeout="30s", + headers={ + Accept="*/*", + ["Content-Type"]="application/json" + }, + body=encodedSendEndpointPayload + }) -pluginOutputData:SetPayload(response["body"]) -pluginOutputData:SetHttpStatusCode(response.status_code) + pluginOutputData:SetPayload(response["body"]) + pluginOutputData:SetHttpStatusCode(response.status_code) +end + +-- optional init function +function init() + +end ``` What the lua script does, is parse the JSON payload from the custom request, extract the `recipient` and the `message` from the payload and the `number` from the URL parameter and call the `/v2/send` endpoint with those parameters. The HTTP status code and the body that is returned by the HTTP request is then returned to the caller (this is done via the `pluginOutputData:SetPayload` and `pluginOutputData:SetHttpStatusCode` functions. diff --git a/plugins/example.def b/plugins/example.def index f4b6dfe..0ce1784 100644 --- a/plugins/example.def +++ b/plugins/example.def @@ -1,2 +1,3 @@ endpoint: my-custom-send-endpoint/:number method: POST +version: 2 diff --git a/plugins/example.lua b/plugins/example.lua index 8a66f11..019a15f 100644 --- a/plugins/example.lua +++ b/plugins/example.lua @@ -1,27 +1,27 @@ local http = require("http") local json = require("json") -local url = "http://127.0.0.1:8080/v2/send" +function exec() + local url = "http://127.0.0.1:8080/v2/send" + local customEndpointPayload = json.decode(pluginInputData.payload) + local sendEndpointPayload = { + recipients = {customEndpointPayload.recipient}, + message = customEndpointPayload.message, + number = pluginInputData.Params.number + } -local customEndpointPayload = json.decode(pluginInputData.payload) + local encodedSendEndpointPayload = json.encode(sendEndpointPayload) + print(encodedSendEndpointPayload) -local sendEndpointPayload = { - recipients = {customEndpointPayload.recipient}, - message = customEndpointPayload.message, - number = pluginInputData.Params.number -} + response, error_message = http.request("POST", url, { + timeout="30s", + headers={ + Accept="*/*", + ["Content-Type"]="application/json" + }, + body=encodedSendEndpointPayload + }) -local encodedSendEndpointPayload = json.encode(sendEndpointPayload) -print(encodedSendEndpointPayload) - -response, error_message = http.request("POST", url, { - timeout="30s", - headers={ - Accept="*/*", - ["Content-Type"]="application/json" - }, - body=encodedSendEndpointPayload -}) - -pluginOutputData:SetPayload(response["body"]) -pluginOutputData:SetHttpStatusCode(response.status_code) + pluginOutputData:SetPayload(response["body"]) + pluginOutputData:SetHttpStatusCode(response.status_code) +end diff --git a/plugins/migrate-v1-plugin-to-v2.md b/plugins/migrate-v1-plugin-to-v2.md new file mode 100644 index 0000000..4512608 --- /dev/null +++ b/plugins/migrate-v1-plugin-to-v2.md @@ -0,0 +1,29 @@ +Migrating a plugin from version `1` to version `2` is really easy. + +* Change your plugin definition (`*.def`) file + and set the version to `2` +e.g: + +``` +endpoint: my-custom-send-endpoint/:number +method: POST +version: 2 +``` + +* Change your plugin script + and implement the `exec` (and optionally the `init`) functions. + +e.g: + +``` +function exec() + -- your plugin code goes here +end + +function init() + -- if your script needs some additional setup (e.g a sqlite database, a config file, etc) + -- the initialization can be done here. +end +``` + + diff --git a/plugins/persistence/README.md b/plugins/persistence/README.md new file mode 100644 index 0000000..5f9e3ff --- /dev/null +++ b/plugins/persistence/README.md @@ -0,0 +1,35 @@ +# Persistence Plugin + +Plugin which writes every received message to a sqlite3 database. + +## Howto enable this plugin + +* Download the `persist-message.def`, `persist-message.lua`, `query-message.def` and `query-message.lua` files and put them in a `plugins` folder on your filesystem +* Create a `persistence` folder on your host system. In this folder the docker container then creates the sqlite3 database. +* Adapt your `docker-compose.yml` to enable the plugin and map the required resources into the docker container + +``` +services: + signal-cli-rest-api: + image: bbernhard/signal-cli-rest-api:latest + environment: + - MODE=json-rpc #supported modes: json-rpc, native, normal (choose the mode you want; the plugin works with all modes) + - ENABLE_PLUGINS=true # enable plugins + - "./plugins:/plugins" #map "plugins" folder from the host system into the docker container + - "./persistence;/persistence" #map "persistence" folder from the host system into the docker container + - RECEIVE_WEBHOOK_URL=http://127.0.0.1:8080/v1/plugins/persistence/persist-message #register an internal webhook endpoint +``` +* Restart your docker container + +Every message that is received is then written to the `messages.db` inside the `persistence` folder. + +The stored messages can then be received via the REST API with: + +`curl -X GET 'http://127.0.0.1:8080/v1/plugins/persistence/query-message'` + +## Debugging and Troubleshooting + +* Make sure that the docker container has write permissions to the `persistence` folder +* On the host system, check if the `messages.db` gets created in the `persistence` folder +* Check the logs. Do you see any error? + diff --git a/plugins/persistence/persist-message.def b/plugins/persistence/persist-message.def new file mode 100644 index 0000000..047e374 --- /dev/null +++ b/plugins/persistence/persist-message.def @@ -0,0 +1,3 @@ +endpoint: persistence/persist-message +method: POST +version: 2 diff --git a/plugins/persistence/persist-message.lua b/plugins/persistence/persist-message.lua new file mode 100644 index 0000000..34f1c3d --- /dev/null +++ b/plugins/persistence/persist-message.lua @@ -0,0 +1,37 @@ +local http = require("http") +local json = require("json") +local sqlite = require("sqlite3").new(); + +function exec() + ok, err = sqlite:open("/persistence/messages.db", { cache = "shared", mode = "rw" }); + if ok then + local data = json.decode(pluginInputData.payload); + if data.params and data.params.envelope and data.params.envelope.dataMessage then + local strippedPayload = json.encode(data.params.envelope) + res, err = sqlite:exec("insert into messages(data) values(?)", strippedPayload) + if err == nil then + pluginOutputData:SetHttpStatusCode(200) + else + pluginOutputData:SetHttpStatusCode(400) + pluginOutputData:SetPayload("Couldn't persist data to sqlite db") + end + else + pluginOutputData:SetHttpStatusCode(200) + end + else + pluginOutputData:SetHttpStatusCode(400) + pluginOutputData:SetPayload("Couldn't persist data to sqlite db") + end +end + +function init() + ok, err = sqlite:open("/persistence/messages.db", { cache = "shared", mode = "rwc" }); + if ok then + res, err = sqlite:exec("create table if not exists messages (id INTEGER PRIMARY KEY, data json, timestamp DATETIME DEFAULT CURRENT_TIMESTAMP)"); + if err ~= nil then + print(err) + return nil, err + end + end + return nil, nil +end diff --git a/plugins/persistence/query-message.def b/plugins/persistence/query-message.def new file mode 100644 index 0000000..00bbec5 --- /dev/null +++ b/plugins/persistence/query-message.def @@ -0,0 +1,3 @@ +endpoint: persistence/query-message +method: GET +version: 2 diff --git a/plugins/persistence/query-message.lua b/plugins/persistence/query-message.lua new file mode 100644 index 0000000..40e2542 --- /dev/null +++ b/plugins/persistence/query-message.lua @@ -0,0 +1,35 @@ +local http = require("http") +local json = require("json") +local sqlite = require("sqlite3").new(); + +function exec() + ok, err = sqlite:open("/persistence/messages.db", { cache = "shared", mode = "rw" }); + if ok then + res, err = sqlite:query("select data, timestamp from messages") + if err == nil then + for _, row in ipairs(res) do + row.data = json.decode(row.data) + end + pluginOutputData:SetPayload(json.encode(res)) + pluginOutputData:SetHttpStatusCode(200) + else + pluginOutputData:SetHttpStatusCode(400) + pluginOutputData:SetPayload("Couldn't query data from sqlite db") + end + else + pluginOutputData:SetHttpStatusCode(400) + pluginOutputData:SetPayload("Couldn't query data from sqlite db") + end +end + +function init() + ok, err = sqlite:open("/persistence/messages.db", { cache = "shared", mode = "rwc" }); + if ok then + res, err = sqlite:exec("create table if not exists messages (id INTEGER PRIMARY KEY, data json, timestamp DATETIME DEFAULT CURRENT_TIMESTAMP)"); + if err ~= nil then + print(err) + return nil, err + end + end + return nil, nil +end diff --git a/src/go.mod b/src/go.mod index a348eed..9f5a509 100644 --- a/src/go.mod +++ b/src/go.mod @@ -3,6 +3,7 @@ module github.com/bbernhard/signal-cli-rest-api go 1.24.0 require ( + github.com/bbernhard/gluasql v0.2.0 github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9 github.com/cyphar/filepath-securejoin v0.2.4 github.com/gabriel-vasile/mimetype v1.4.8 @@ -17,13 +18,14 @@ require ( github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.4 github.com/tidwall/sjson v1.2.5 - github.com/yuin/gopher-lua v1.1.1 + github.com/yuin/gopher-lua v1.1.2 gopkg.in/yaml.v2 v2.4.0 layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf layeh.com/gopher-luar v1.0.11 ) require ( + filippo.io/edwards25519 v1.2.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/bytedance/sonic v1.12.8 // indirect github.com/bytedance/sonic/loader v0.2.3 // indirect @@ -36,13 +38,16 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.24.0 // indirect + github.com/go-sql-driver/mysql v1.10.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/junhsieh/goexamples v0.0.0-20210908032526-acdd3160140b // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-sqlite3 v1.14.44 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect diff --git a/src/main.go b/src/main.go index 2450496..ede0cc1 100644 --- a/src/main.go +++ b/src/main.go @@ -349,6 +349,18 @@ func main() { } for _, pluginConfig := range pluginConfigs.Configs { + if pluginConfig.Version > 1 { + err = pluginHandler.InitPlugin(pluginConfig) + if err != nil { + log.Error("Couldn't initialize plugin ", pluginConfig.Endpoint) + continue + } + } else { + log.Info("Plugin ", pluginConfig.Endpoint, " still uses plugin version 1. Consider migrating to version 2! (see https://github.com/bbernhard/signal-cli-rest-api/plugins/migrate-v1-plugin-to-v2.md)") + } + + log.Info("Registering plugin ", pluginConfig.Endpoint) + if pluginConfig.Method == "GET" { plugins.GET(pluginConfig.Endpoint, pluginHandler.ExecutePlugin(pluginConfig)) } else if pluginConfig.Method == "POST" { diff --git a/src/plugin_loader.go b/src/plugin_loader.go index fa11447..a4e1f40 100644 --- a/src/plugin_loader.go +++ b/src/plugin_loader.go @@ -1,27 +1,30 @@ package main import ( - "github.com/yuin/gopher-lua" - "github.com/cjoudrey/gluahttp" - "layeh.com/gopher-luar" - luajson "layeh.com/gopher-json" - "github.com/gin-gonic/gin" + "errors" "io" - log "github.com/sirupsen/logrus" - "github.com/bbernhard/signal-cli-rest-api/utils" - "github.com/bbernhard/signal-cli-rest-api/api" - "strings" "net/http" + "strings" + + gluasql "github.com/bbernhard/gluasql" + "github.com/bbernhard/signal-cli-rest-api/api" + "github.com/bbernhard/signal-cli-rest-api/utils" + "github.com/cjoudrey/gluahttp" + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" + lua "github.com/yuin/gopher-lua" + luajson "layeh.com/gopher-json" + luar "layeh.com/gopher-luar" ) type PluginInputData struct { - Params map[string]string + Params map[string]string QueryParams map[string]string - Payload string + Payload string } type PluginOutputData struct { - payload string + payload string httpStatusCode int } @@ -41,7 +44,7 @@ func (p *PluginOutputData) HttpStatusCode() int { return p.httpStatusCode } -func execPlugin(c *gin.Context, pluginConfig utils.PluginConfig) { +func execPluginV1(c *gin.Context, pluginConfig utils.PluginConfig) { jsonData, err := io.ReadAll(c.Request.Body) if err != nil { c.JSON(400, api.Error{Msg: "Couldn't process request - invalid input data"}) @@ -50,13 +53,13 @@ func execPlugin(c *gin.Context, pluginConfig utils.PluginConfig) { } pluginInputData := &PluginInputData{ - Params: make(map[string]string), + Params: make(map[string]string), QueryParams: make(map[string]string), - Payload: string(jsonData), + Payload: string(jsonData), } pluginOutputData := &PluginOutputData{ - payload: "", + payload: "", httpStatusCode: 200, } @@ -78,8 +81,10 @@ func execPlugin(c *gin.Context, pluginConfig utils.PluginConfig) { l.SetGlobal("pluginOutputData", luar.New(l, pluginOutputData)) l.PreloadModule("http", gluahttp.NewHttpModule(&http.Client{}).Loader) luajson.Preload(l) + gluasql.Preload(l) defer l.Close() if err := l.DoFile(pluginConfig.ScriptPath); err != nil { + log.Error("Error executing lua script: ", err) c.JSON(400, api.Error{Msg: err.Error()}) return } @@ -87,16 +92,138 @@ func execPlugin(c *gin.Context, pluginConfig utils.PluginConfig) { c.JSON(pluginOutputData.HttpStatusCode(), pluginOutputData.Payload()) } +func execPluginV2(c *gin.Context, pluginConfig utils.PluginConfig) { + jsonData, err := io.ReadAll(c.Request.Body) + if err != nil { + c.JSON(400, api.Error{Msg: "Couldn't process request - invalid input data"}) + log.Error(err.Error()) + return + } + + pluginInputData := &PluginInputData{ + Params: make(map[string]string), + QueryParams: make(map[string]string), + Payload: string(jsonData), + } + + pluginOutputData := &PluginOutputData{ + payload: "", + httpStatusCode: 200, + } + + parts := strings.Split(pluginConfig.Endpoint, "/") + for _, part := range parts { + if strings.HasPrefix(part, ":") { + paramName := strings.TrimPrefix(part, ":") + pluginInputData.Params[paramName] = c.Param(paramName) + } + } + + queryParams := c.Request.URL.Query() + for key, values := range queryParams { + pluginInputData.QueryParams[key] = values[0] + } + + l := lua.NewState() + l.SetGlobal("pluginInputData", luar.New(l, pluginInputData)) + l.SetGlobal("pluginOutputData", luar.New(l, pluginOutputData)) + l.PreloadModule("http", gluahttp.NewHttpModule(&http.Client{}).Loader) + luajson.Preload(l) + gluasql.Preload(l) + defer l.Close() + if err := l.DoFile(pluginConfig.ScriptPath); err != nil { + log.Error("Error executing lua script: ", err) + c.JSON(400, api.Error{Msg: err.Error()}) + return + } + + // Get global "exec" + lv := l.GetGlobal("exec") + + // Check if it exists and is a function + if fn, ok := lv.(*lua.LFunction); ok { + err := l.CallByParam(lua.P{ + Fn: fn, + NRet: 1, // exec function returns one value + Protect: true, + }) + + if err != nil { + log.Error("Couldn't execute plugin: ", err.Error()) + c.JSON(400, "Couldn't execute plugin: "+err.Error()) + return + } + + ret := l.Get(-1) + l.Pop(1) + + if ret != lua.LNil { + log.Error("Couldn't execute plugin") + c.JSON(400, "Couldn't execute plugin") + } + c.Data( + pluginOutputData.HttpStatusCode(), + "application/json", + []byte(pluginOutputData.Payload()), + ) + } else { + log.Error("Couldn't execute plugin. No exec function implemented!") + c.JSON(400, "Couldn't execute plugin. No exec function implemented!") + } +} + type plugHandler struct { } func (p plugHandler) ExecutePlugin(pluginConfig utils.PluginConfig) gin.HandlerFunc { fn := func(c *gin.Context) { - execPlugin(c, pluginConfig) + if pluginConfig.Version == 1 { + execPluginV1(c, pluginConfig) + } else { + execPluginV2(c, pluginConfig) + } } return gin.HandlerFunc(fn) } -//exported +func (p plugHandler) InitPlugin(pluginConfig utils.PluginConfig) error { + l := lua.NewState() + l.PreloadModule("http", gluahttp.NewHttpModule(&http.Client{}).Loader) + luajson.Preload(l) + gluasql.Preload(l) + defer l.Close() + err := l.DoFile(pluginConfig.ScriptPath) + if err != nil { + log.Error("Error executing lua script: ", err) + } + + // Get global "init" + lv := l.GetGlobal("init") + + // Check if it exists and is a function + if fn, ok := lv.(*lua.LFunction); ok { + err := l.CallByParam(lua.P{ + Fn: fn, + NRet: 2, // init function returns two values + Protect: true, + }) + + if err != nil { + return err + } + + _ = l.Get(-2) + errVal := l.Get(-1) + l.Pop(2) + + if errVal != lua.LNil { + return errors.New("Couldn't initialize lua script: " + errVal.String()) + } + } + + return nil +} + +// exported var PluginHandler plugHandler diff --git a/src/utils/plugin_config.go b/src/utils/plugin_config.go index 5349216..e752b95 100644 --- a/src/utils/plugin_config.go +++ b/src/utils/plugin_config.go @@ -1,16 +1,18 @@ package utils import ( - "gopkg.in/yaml.v2" "io/ioutil" "os" "path/filepath" "strings" + + "gopkg.in/yaml.v2" ) type PluginConfig struct { Endpoint string `yaml:"endpoint"` Method string `yaml:"method"` + Version int `yaml:"version,omitempty"` ScriptPath string } @@ -40,6 +42,7 @@ func (c *PluginConfigs) Load(baseDirectory string) error { } var pluginConfig PluginConfig + pluginConfig.Version = 1 err = yaml.Unmarshal(data, &pluginConfig) if err != nil { return err diff --git a/src/utils/plugin_handler.go b/src/utils/plugin_handler.go index d2931b6..b432efd 100644 --- a/src/utils/plugin_handler.go +++ b/src/utils/plugin_handler.go @@ -6,4 +6,5 @@ import ( type PluginHandler interface { ExecutePlugin(pluginConfig PluginConfig) gin.HandlerFunc + InitPlugin(pluginConfig PluginConfig) error }