futriix/tests/unit/moduleapi/scriptingengine.tcl
Ricardo Dias 6adef8e2f9
Adds support for scripting engines as Valkey modules (#1277)
This PR extends the module API to support the addition of different
scripting engines to execute user defined functions.

The scripting engine can be implemented as a Valkey module, and can be
dynamically loaded with the `loadmodule` config directive, or with the
`MODULE LOAD` command.

This PR also adds an example of a dummy scripting engine module, to show
how to use the new module API. The dummy module is implemented in
`tests/modules/helloscripting.c`.

The current module API support, only allows to load scripting engines to
run functions using `FCALL` command.

The additions to the module API are the following:

```c
/* This struct represents a scripting engine function that results from the
 * compilation of a script by the engine implementation. */
struct ValkeyModuleScriptingEngineCompiledFunction

typedef ValkeyModuleScriptingEngineCompiledFunction **(*ValkeyModuleScriptingEngineCreateFunctionsLibraryFunc)(
    ValkeyModuleScriptingEngineCtx *engine_ctx,
    const char *code,
    size_t timeout,
    size_t *out_num_compiled_functions,
    char **err);

typedef void (*ValkeyModuleScriptingEngineCallFunctionFunc)(
    ValkeyModuleCtx *module_ctx,
    ValkeyModuleScriptingEngineCtx *engine_ctx,
    ValkeyModuleScriptingEngineFunctionCtx *func_ctx,
    void *compiled_function,
    ValkeyModuleString **keys,
    size_t nkeys,
    ValkeyModuleString **args,
    size_t nargs);

typedef size_t (*ValkeyModuleScriptingEngineGetUsedMemoryFunc)(
    ValkeyModuleScriptingEngineCtx *engine_ctx);

typedef size_t (*ValkeyModuleScriptingEngineGetFunctionMemoryOverheadFunc)(
    void *compiled_function);

typedef size_t (*ValkeyModuleScriptingEngineGetEngineMemoryOverheadFunc)(
    ValkeyModuleScriptingEngineCtx *engine_ctx);

typedef void (*ValkeyModuleScriptingEngineFreeFunctionFunc)(
    ValkeyModuleScriptingEngineCtx *engine_ctx,
    void *compiled_function);

/* This struct stores the callback functions implemented by the scripting
 * engine to provide the functionality for the `FUNCTION *` commands. */
typedef struct ValkeyModuleScriptingEngineMethodsV1 {
    uint64_t version; /* Version of this structure for ABI compat. */

    /* Library create function callback. When a new script is loaded, this
     * callback will be called with the script code, and returns a list of
     * ValkeyModuleScriptingEngineCompiledFunc objects. */
    ValkeyModuleScriptingEngineCreateFunctionsLibraryFunc create_functions_library;

    /* The callback function called when `FCALL` command is called on a function
     * registered in this engine. */
    ValkeyModuleScriptingEngineCallFunctionFunc call_function;

    /* Function callback to get current used memory by the engine. */
    ValkeyModuleScriptingEngineGetUsedMemoryFunc get_used_memory;

    /* Function callback to return memory overhead for a given function. */
    ValkeyModuleScriptingEngineGetFunctionMemoryOverheadFunc get_function_memory_overhead;

    /* Function callback to return memory overhead of the engine. */
    ValkeyModuleScriptingEngineGetEngineMemoryOverheadFunc get_engine_memory_overhead;

    /* Function callback to free the memory of a registered engine function. */
    ValkeyModuleScriptingEngineFreeFunctionFunc free_function;
} ValkeyModuleScriptingEngineMethodsV1;

/* Registers a new scripting engine in the server.
 *
 * - `engine_name`: the name of the scripting engine. This name will match
 *   against the engine name specified in the script header using a shebang.
 *
 * - `engine_ctx`: engine specific context pointer.
 *
 * - `engine_methods`: the struct with the scripting engine callback functions
 * pointers.
 */
int ValkeyModule_RegisterScriptingEngine(ValkeyModuleCtx *ctx,
                                         const char *engine_name,
                                         void *engine_ctx,
                                         ValkeyModuleScriptingEngineMethods engine_methods);

/* Removes the scripting engine from the server.
 *
 * `engine_name` is the name of the scripting engine.
 *
 */
int ValkeyModule_UnregisterScriptingEngine(ValkeyModuleCtx *ctx, const char *engine_name);
```

---------

Signed-off-by: Ricardo Dias <ricardo.dias@percona.com>
2024-12-21 23:09:35 +01:00

127 lines
4.4 KiB
Tcl

set testmodule [file normalize tests/modules/helloscripting.so]
set HELLO_PROGRAM "#!hello name=mylib\nFUNCTION foo\nARGS 0\nRETURN\nFUNCTION bar\nCONSTI 432\nRETURN"
start_server {tags {"modules"}} {
r module load $testmodule
r function load $HELLO_PROGRAM
test {Load script with invalid library name} {
assert_error {ERR Library names can only contain letters, numbers, or underscores(_) and must be at least one character long} {r function load "#!hello name=my-lib\nFUNCTION foo\nARGS 0\nRETURN"}
}
test {Load script with existing library} {
assert_error {ERR Library 'mylib' already exists} {r function load $HELLO_PROGRAM}
}
test {Load script with invalid engine} {
assert_error {ERR Engine 'wasm' not found} {r function load "#!wasm name=mylib2\nFUNCTION foo\nARGS 0\nRETURN"}
}
test {Load script with no functions} {
assert_error {ERR No functions registered} {r function load "#!hello name=mylib2\n"}
}
test {Load script with duplicate function} {
assert_error {ERR Function foo already exists} {r function load "#!hello name=mylib2\nFUNCTION foo\nARGS 0\nRETURN"}
}
test {Load script with no metadata header} {
assert_error {ERR Missing library metadata} {r function load "FUNCTION foo\nARGS 0\nRETURN"}
}
test {Load script with header without lib name} {
assert_error {ERR Library name was not given} {r function load "#!hello \n"}
}
test {Load script with header with unknown param} {
assert_error {ERR Invalid metadata value given: nme=mylib} {r function load "#!hello nme=mylib\n"}
}
test {Load script with header with lib name passed twice} {
assert_error {ERR Invalid metadata value, name argument was given multiple times} {r function load "#!hello name=mylib2 name=mylib3\n"}
}
test {Load script with invalid function name} {
assert_error {ERR Function names can only contain letters, numbers, or underscores(_) and must be at least one character long} {r function load "#!hello name=mylib2\nFUNCTION foo-bar\nARGS 0\nRETURN"}
}
test {Load script with duplicate function} {
assert_error {ERR Function already exists in the library} {r function load "#!hello name=mylib2\nFUNCTION foo\nARGS 0\nRETURN\nFUNCTION foo\nARGS 0\nRETURN"}
}
test {Call scripting engine function: calling foo works} {
r fcall foo 0 134
} {134}
test {Call scripting engine function: calling bar works} {
r fcall bar 0
} {432}
test {Replace function library and call functions} {
set result [r function load replace "#!hello name=mylib\nFUNCTION foo\nARGS 0\nRETURN\nFUNCTION bar\nCONSTI 500\nRETURN"]
assert_equal $result "mylib"
set result [r fcall foo 0 132]
assert_equal $result 132
set result [r fcall bar 0]
assert_equal $result 500
}
test {List scripting engine functions} {
r function load replace "#!hello name=mylib\nFUNCTION foobar\nARGS 0\nRETURN"
r function list
} {{library_name mylib engine HELLO functions {{name foobar description {} flags {}}}}}
test {Load a second library and call a function} {
r function load "#!hello name=mylib2\nFUNCTION getarg\nARGS 0\nRETURN"
set result [r fcall getarg 0 456]
assert_equal $result 456
}
test {Delete all libraries and functions} {
set result [r function flush]
assert_equal $result {OK}
r function list
} {}
test {Test the deletion of a single library} {
r function load $HELLO_PROGRAM
r function load "#!hello name=mylib2\nFUNCTION getarg\nARGS 0\nRETURN"
set result [r function delete mylib]
assert_equal $result {OK}
set result [r fcall getarg 0 446]
assert_equal $result 446
}
test {Test dump and restore function library} {
r function load $HELLO_PROGRAM
set result [r fcall bar 0]
assert_equal $result 432
set dump [r function dump]
set result [r function flush]
assert_equal $result {OK}
set result [r function restore $dump]
assert_equal $result {OK}
set result [r fcall getarg 0 436]
assert_equal $result 436
set result [r fcall bar 0]
assert_equal $result 432
}
test {Unload scripting engine module} {
set result [r module unload helloengine]
assert_equal $result "OK"
}
}