diff --git a/src/commands.c b/src/commands.c index 8eed44e01..2183d09a1 100644 --- a/src/commands.c +++ b/src/commands.c @@ -6612,8 +6612,8 @@ struct redisCommand redisCommandTable[] = { {"evalsha","Execute a Lua script server side","Depends on the script that is executed.","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVALSHA_History,EVALSHA_Hints,evalShaCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_WRITE|CMD_KEY_READ,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVALSHA_Args}, {"evalsha_ro","Execute a read-only Lua script server side","Depends on the script that is executed.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVALSHA_RO_History,EVALSHA_RO_Hints,evalShaRoCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_READ,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVALSHA_RO_Args}, {"eval_ro","Execute a read-only Lua script server side","Depends on the script that is executed.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVAL_RO_History,EVAL_RO_Hints,evalRoCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_READ,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVAL_RO_Args}, -{"fcall","PATCH__TBD__38__","PATCH__TBD__37__","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FCALL_History,FCALL_Hints,fcallCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_WRITE|CMD_KEY_READ,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},functionGetKeys,.args=FCALL_Args}, -{"fcall_ro","PATCH__TBD__7__","PATCH__TBD__6__","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FCALL_RO_History,FCALL_RO_Hints,fcallroCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_READ,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},functionGetKeys,.args=FCALL_RO_Args}, +{"fcall","PATCH__TBD__38__","PATCH__TBD__37__","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FCALL_History,FCALL_Hints,fcallCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_WRITE|CMD_KEY_READ,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},functionGetKeys,.args=FCALL_Args}, +{"fcall_ro","PATCH__TBD__7__","PATCH__TBD__6__","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FCALL_RO_History,FCALL_RO_Hints,fcallroCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_READ,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},functionGetKeys,.args=FCALL_RO_Args}, {"function","A container for function commands","Depends on subcommand.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FUNCTION_History,FUNCTION_Hints,NULL,-2,0,0,.subcommands=FUNCTION_Subcommands}, {"script","A container for Lua scripts management commands","Depends on subcommand.","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,SCRIPT_History,SCRIPT_Hints,NULL,-2,0,0,.subcommands=SCRIPT_Subcommands}, /* sentinel */ diff --git a/src/commands/fcall.json b/src/commands/fcall.json index e5e63c638..e5dfc3898 100644 --- a/src/commands/fcall.json +++ b/src/commands/fcall.json @@ -11,7 +11,8 @@ "NOSCRIPT", "SKIP_MONITOR", "MAY_REPLICATE", - "NO_MANDATORY_KEYS" + "NO_MANDATORY_KEYS", + "STALE" ], "acl_categories": [ "SCRIPTING" diff --git a/src/commands/fcall_ro.json b/src/commands/fcall_ro.json index 4d58b2f9a..ae0bf61e3 100644 --- a/src/commands/fcall_ro.json +++ b/src/commands/fcall_ro.json @@ -10,7 +10,8 @@ "command_flags": [ "NOSCRIPT", "SKIP_MONITOR", - "NO_MANDATORY_KEYS" + "NO_MANDATORY_KEYS", + "STALE" ], "acl_categories": [ "SCRIPTING" diff --git a/src/function_lua.c b/src/function_lua.c index e6b8a2727..3dbc8419e 100644 --- a/src/function_lua.c +++ b/src/function_lua.c @@ -68,6 +68,13 @@ typedef struct loadCtx { monotime start_time; } loadCtx; +typedef struct registerFunctionArgs { + sds name; + sds desc; + luaFunctionCtx *lua_f_ctx; + uint64_t f_flags; +} registerFunctionArgs; + /* Hook for FUNCTION LOAD execution. * Used to cancel the execution in case of a timeout (500ms). * This execution should be fast and should only register @@ -224,56 +231,214 @@ static void luaEngineFreeFunction(void *engine_ctx, void *compiled_function) { zfree(f_ctx); } -static int luaRegisterFunction(lua_State *lua) { - int argc = lua_gettop(lua); - if (argc < 2 || argc > 3) { - luaPushError(lua, "wrong number of arguments to redis.register_function"); - return luaRaiseError(lua); +static void luaRegisterFunctionArgsInitialize(registerFunctionArgs *register_f_args, + sds name, + sds desc, + luaFunctionCtx *lua_f_ctx, + uint64_t flags) +{ + *register_f_args = (registerFunctionArgs){ + .name = name, + .desc = desc, + .lua_f_ctx = lua_f_ctx, + .f_flags = flags, + }; +} + +static void luaRegisterFunctionArgsDispose(lua_State *lua, registerFunctionArgs *register_f_args) { + sdsfree(register_f_args->name); + if (register_f_args->desc) sdsfree(register_f_args->desc); + lua_unref(lua, register_f_args->lua_f_ctx->lua_function_ref); + zfree(register_f_args->lua_f_ctx); +} + +/* Read function flags located on the top of the Lua stack. + * On success, return C_OK and set the flags to 'flags' out parameter + * Return C_ERR if encounter an unknown flag. */ +static int luaRegisterFunctionReadFlags(lua_State *lua, uint64_t *flags) { + int j = 1; + int ret = C_ERR; + int f_flags = 0; + while(1) { + lua_pushnumber(lua,j++); + lua_gettable(lua,-2); + int t = lua_type(lua,-1); + if (t == LUA_TNIL) { + lua_pop(lua,1); + break; + } + if (!lua_isstring(lua, -1)) { + lua_pop(lua,1); + goto done; + } + + const char *flag_str = lua_tostring(lua, -1); + int found = 0; + for (scriptFlag *flag = scripts_flags_def; flag->str ; ++flag) { + if (!strcasecmp(flag->str, flag_str)) { + f_flags |= flag->flag; + found = 1; + break; + } + } + /* pops the value to continue the iteration */ + lua_pop(lua,1); + if (!found) { + /* flag not found */ + goto done; + } } + + *flags = f_flags; + ret = C_OK; + +done: + return ret; +} + +static int luaRegisterFunctionReadNamedArgs(lua_State *lua, registerFunctionArgs *register_f_args) { + char *err = NULL; + sds name = NULL; + sds desc = NULL; + luaFunctionCtx *lua_f_ctx = NULL; + uint64_t flags = 0; + if (!lua_istable(lua, 1)) { + err = "calling redis.register_function with a single argument is only applicable to Lua table (representing named arguments)."; + goto error; + } + + /* Iterating on all the named arguments */ + lua_pushnil(lua); + while (lua_next(lua, -2)) { + /* Stack now: table, key, value */ + if (!lua_isstring(lua, -2)) { + err = "named argument key given to redis.register_function is not a string"; + goto error; + } + const char *key = lua_tostring(lua, -2); + if (!strcasecmp(key, "function_name")) { + if (!(name = luaGetStringSds(lua, -1))) { + err = "function_name argument given to redis.register_function must be a string"; + goto error; + } + } else if (!strcasecmp(key, "description")) { + if (!(desc = luaGetStringSds(lua, -1))) { + err = "description argument given to redis.register_function must be a string"; + goto error; + } + } else if (!strcasecmp(key, "callback")) { + if (!lua_isfunction(lua, -1)) { + err = "callback argument given to redis.register_function must be a function"; + goto error; + } + int lua_function_ref = luaL_ref(lua, LUA_REGISTRYINDEX); + + lua_f_ctx = zmalloc(sizeof(*lua_f_ctx)); + lua_f_ctx->lua_function_ref = lua_function_ref; + continue; /* value was already popped, so no need to pop it out. */ + } else if (!strcasecmp(key, "flags")) { + if (!lua_istable(lua, -1)) { + err = "flags argument to redis.register_function must be a table representing function flags"; + goto error; + } + if (luaRegisterFunctionReadFlags(lua, &flags) != C_OK) { + err = "unknown flag given"; + goto error; + } + } else { + /* unknown argument was given, raise an error */ + err = "unknown argument given to redis.register_function"; + goto error; + } + lua_pop(lua, 1); /* pop the value to continue the iteration */ + } + + if (!name) { + err = "redis.register_function must get a function name argument"; + goto error; + } + + if (!lua_f_ctx) { + err = "redis.register_function must get a callback argument"; + goto error; + } + + luaRegisterFunctionArgsInitialize(register_f_args, name, desc, lua_f_ctx, flags); + + return C_OK; + +error: + if (name) sdsfree(name); + if (desc) sdsfree(desc); + if (lua_f_ctx) { + lua_unref(lua, lua_f_ctx->lua_function_ref); + zfree(lua_f_ctx); + } + luaPushError(lua, err); + return C_ERR; +} + +static int luaRegisterFunctionReadPositionalArgs(lua_State *lua, registerFunctionArgs *register_f_args) { + char *err = NULL; + sds name = NULL; + sds desc = NULL; + luaFunctionCtx *lua_f_ctx = NULL; + if (!(name = luaGetStringSds(lua, 1))) { + err = "first argument to redis.register_function must be a string"; + goto error; + } + + if (!lua_isfunction(lua, 2)) { + err = "second argument to redis.register_function must be a function"; + goto error; + } + + int lua_function_ref = luaL_ref(lua, LUA_REGISTRYINDEX); + + lua_f_ctx = zmalloc(sizeof(*lua_f_ctx)); + lua_f_ctx->lua_function_ref = lua_function_ref; + + luaRegisterFunctionArgsInitialize(register_f_args, name, NULL, lua_f_ctx, 0); + + return C_OK; + +error: + if (name) sdsfree(name); + if (desc) sdsfree(desc); + luaPushError(lua, err); + return C_ERR; +} + +static int luaRegisterFunctionReadArgs(lua_State *lua, registerFunctionArgs *register_f_args) { + int argc = lua_gettop(lua); + if (argc < 1 || argc > 2) { + luaPushError(lua, "wrong number of arguments to redis.register_function"); + return C_ERR; + } + + if (argc == 1) { + return luaRegisterFunctionReadNamedArgs(lua, register_f_args); + } else { + return luaRegisterFunctionReadPositionalArgs(lua, register_f_args); + } +} + +static int luaRegisterFunction(lua_State *lua) { + registerFunctionArgs register_f_args = {0}; + loadCtx *load_ctx = luaGetFromRegistry(lua, REGISTRY_LOAD_CTX_NAME); if (!load_ctx) { luaPushError(lua, "redis.register_function can only be called on FUNCTION LOAD command"); return luaRaiseError(lua); } - if (!lua_isstring(lua, 1)) { - luaPushError(lua, "first argument to redis.register_function must be a string"); + if (luaRegisterFunctionReadArgs(lua, ®ister_f_args) != C_OK) { return luaRaiseError(lua); } - if (!lua_isfunction(lua, 2)) { - luaPushError(lua, "second argument to redis.register_function must be a function"); - return luaRaiseError(lua); - } - - if (argc == 3 && !lua_isstring(lua, 3)) { - luaPushError(lua, "third argument to redis.register_function must be a string"); - return luaRaiseError(lua); - } - - size_t function_name_len; - const char *function_name = lua_tolstring(lua, 1, &function_name_len); - sds function_name_sds = sdsnewlen(function_name, function_name_len); - - sds desc_sds = NULL; - if (argc == 3){ - size_t desc_len; - const char *desc = lua_tolstring(lua, 3, &desc_len); - desc_sds = sdsnewlen(desc, desc_len); - lua_pop(lua, 1); /* pop out the description */ - } - - int lua_function_ref = luaL_ref(lua, LUA_REGISTRYINDEX); - - luaFunctionCtx *lua_f_ctx = zmalloc(sizeof(*lua_f_ctx)); - *lua_f_ctx = (luaFunctionCtx ) { .lua_function_ref = lua_function_ref, }; - sds err = NULL; - if (functionLibCreateFunction(function_name_sds, lua_f_ctx, load_ctx->li, desc_sds, &err) != C_OK) { - sdsfree(function_name_sds); - if (desc_sds) sdsfree(desc_sds); - lua_unref(lua, lua_f_ctx->lua_function_ref); - zfree(lua_f_ctx); + if (functionLibCreateFunction(register_f_args.name, register_f_args.lua_f_ctx, load_ctx->li, register_f_args.desc, register_f_args.f_flags, &err) != C_OK) { + luaRegisterFunctionArgsDispose(lua, ®ister_f_args); luaPushError(lua, err); sdsfree(err); return luaRaiseError(lua); @@ -298,6 +463,7 @@ int luaEngineInitEngine() { lua_settable(lua_engine_ctx->lua, -3); luaRegisterLogFunction(lua_engine_ctx->lua); + luaRegisterVersion(lua_engine_ctx->lua); lua_settable(lua_engine_ctx->lua, LUA_REGISTRYINDEX); diff --git a/src/functions.c b/src/functions.c index 5a8cd27c1..76e8c21d7 100644 --- a/src/functions.c +++ b/src/functions.c @@ -198,7 +198,7 @@ functionsLibCtx* functionsLibCtxCreate() { * the function will verify that the given name is following the naming format * and return an error if its not. */ -int functionLibCreateFunction(sds name, void *function, functionLibInfo *li, sds desc, sds *err) { +int functionLibCreateFunction(sds name, void *function, functionLibInfo *li, sds desc, uint64_t f_flags, sds *err) { if (functionsVerifyName(name) != C_OK) { *err = sdsnew("Function names can only contain letters and numbers and must be at least one character long"); return C_ERR; @@ -215,6 +215,7 @@ int functionLibCreateFunction(sds name, void *function, functionLibInfo *li, sds .function = function, .li = li, .desc = desc, + .f_flags = f_flags, }; int res = dictAdd(li->functions, fi->name, fi); @@ -410,6 +411,24 @@ void functionStatsCommand(client *c) { dictReleaseIterator(iter); } +static void functionListReplyFlags(client *c, functionInfo *fi) { + /* First count the number of flags we have */ + int flagcount = 0; + for (scriptFlag *flag = scripts_flags_def; flag->str ; ++flag) { + if (fi->f_flags & flag->flag) { + ++flagcount; + } + } + + addReplySetLen(c, flagcount); + + for (scriptFlag *flag = scripts_flags_def; flag->str ; ++flag) { + if (fi->f_flags & flag->flag) { + addReplyStatus(c, flag->str); + } + } +} + /* * FUNCTION LIST [LIBRARYNAME PATTERN] [WITHCODE] * @@ -480,7 +499,7 @@ void functionListCommand(client *c) { dictEntry *function_entry = NULL; while ((function_entry = dictNext(functions_iter))) { functionInfo *fi = dictGetVal(function_entry); - addReplyMapLen(c, 2); + addReplyMapLen(c, 3); addReplyBulkCString(c, "name"); addReplyBulkCBuffer(c, fi->name, sdslen(fi->name)); addReplyBulkCString(c, "description"); @@ -489,6 +508,8 @@ void functionListCommand(client *c) { } else { addReplyNull(c); } + addReplyBulkCString(c, "flags"); + functionListReplyFlags(c, fi); } dictReleaseIterator(functions_iter); @@ -549,12 +570,69 @@ static void fcallCommandGeneric(client *c, int ro) { return; } + if ((fi->f_flags & SCRIPT_FLAG_NO_CLUSTER) && server.cluster_enabled) { + addReplyError(c, "Can not run function on cluster, 'no-cluster' flag is set."); + return; + } + + if (!(fi->f_flags & SCRIPT_FLAG_ALLOW_OOM) && server.script_oom && server.maxmemory) { + addReplyError(c, "-OOM allow-oom flag is not set on the function, " + "can not run it when used memory > 'maxmemory'"); + return; + } + + if (server.masterhost && server.repl_state != REPL_STATE_CONNECTED && + server.repl_serve_stale_data == 0 && !(fi->f_flags & SCRIPT_FLAG_ALLOW_STALE)) + { + addReplyError(c, "-MASTERDOWN Link with MASTER is down, " + "replica-serve-stale-data is set to 'no' " + "and 'allow-stale' flag is not set on the function."); + return; + } + + if (!(fi->f_flags & SCRIPT_FLAG_NO_WRITES)) { + /* Function may perform writes we need to verify: + * 1. we are not a readonly replica + * 2. no disk error detected + * 3. command is not 'fcall_ro' */ + if (server.masterhost && server.repl_slave_ro && c->id != CLIENT_ID_AOF + && !(c->flags & CLIENT_MASTER)) + { + addReplyError(c, "Can not run a function with write flag on readonly replica"); + return; + } + + int deny_write_type = writeCommandsDeniedByDiskError(); + if (deny_write_type != DISK_ERROR_TYPE_NONE && server.masterhost == NULL) { + if (deny_write_type == DISK_ERROR_TYPE_RDB) + addReplyError(c, "-MISCONF Redis is configured to save RDB snapshots, " + "but it is currently not able to persist on disk. " + "So its impossible to run functions that has 'write' flag on."); + else + addReplyErrorFormat(c, "-MISCONF Redis is configured to persist data to AOF, " + "but it is currently not able to persist on disk. " + "So its impossible to run functions that has 'write' flag on. " + "AOF error: %s", strerror(server.aof_last_write_errno)); + return; + } + + if (ro) { + addReplyError(c, "Can not execute a function with write flag using fcall_ro."); + return; + } + } + scriptRunCtx run_ctx; scriptPrepareForRun(&run_ctx, fi->li->ei->c, c, fi->name); - if (ro) { + if (ro || (fi->f_flags & SCRIPT_FLAG_NO_WRITES)) { + /* On fcall_ro or on functions that do not have the 'write' + * flag, we will not allow write commands. */ run_ctx.flags |= SCRIPT_READ_ONLY; } + if (fi->f_flags & SCRIPT_FLAG_ALLOW_OOM) { + run_ctx.flags |= SCRIPT_ALLOW_OOM; + } engine->call(&run_ctx, engine->engine_ctx, fi->function, c->argv + 3, numkeys, c->argv + 3 + numkeys, c->argc - 3 - numkeys); scriptResetRun(&run_ctx); diff --git a/src/functions.h b/src/functions.h index f49040193..fb2b74de9 100644 --- a/src/functions.h +++ b/src/functions.h @@ -96,6 +96,7 @@ typedef struct functionInfo { to run the function, usually it's the function compiled code. */ functionLibInfo* li; /* Pointer to the library created the function */ sds desc; /* Function description */ + uint64_t f_flags; /* Function flags */ } functionInfo; /* Hold information about the specific library. @@ -124,7 +125,7 @@ void functionsLibCtxFree(functionsLibCtx *lib_ctx); void functionsLibCtxClear(functionsLibCtx *lib_ctx); void functionsLibCtxSwapWithCurrent(functionsLibCtx *lib_ctx); -int functionLibCreateFunction(sds name, void *function, functionLibInfo *li, sds desc, sds *err); +int functionLibCreateFunction(sds name, void *function, functionLibInfo *li, sds desc, uint64_t f_flags, sds *err); int luaEngineInitEngine(); int functionsInit(); diff --git a/src/script.c b/src/script.c index bef4f383c..8b1545b1a 100644 --- a/src/script.c +++ b/src/script.c @@ -31,6 +31,14 @@ #include "script.h" #include "cluster.h" +scriptFlag scripts_flags_def[] = { + {.flag = SCRIPT_FLAG_NO_WRITES, .str = "no-writes"}, + {.flag = SCRIPT_FLAG_ALLOW_OOM, .str = "allow-oom"}, + {.flag = SCRIPT_FLAG_ALLOW_STALE, .str = "allow-stale"}, + {.flag = SCRIPT_FLAG_NO_CLUSTER, .str = "no-cluster"}, + {.flag = 0, .str = NULL}, /* flags array end */ +}; + /* On script invocation, holding the current run context */ static scriptRunCtx *curr_run_ctx = NULL; @@ -281,6 +289,11 @@ static int scriptVerifyWriteCommandAllow(scriptRunCtx *run_ctx, char **err) { } static int scriptVerifyOOM(scriptRunCtx *run_ctx, char **err) { + if (run_ctx->flags & SCRIPT_ALLOW_OOM) { + /* Allow running any command even if OOM reached */ + return C_OK; + } + /* If we reached the memory limit configured via maxmemory, commands that * could enlarge the memory usage are not allowed, but only if this is the * first write in the context of this script, otherwise we can't stop @@ -348,6 +361,32 @@ int scriptSetRepl(scriptRunCtx *run_ctx, int repl) { return C_OK; } +static int scriptVerifyAllowStale(client *c, sds *err) { + if (!server.masterhost) { + /* Not a replica, stale is irrelevant */ + return C_OK; + } + + if (server.repl_state == REPL_STATE_CONNECTED) { + /* Connected to replica, stale is irrelevant */ + return C_OK; + } + + if (server.repl_serve_stale_data == 1) { + /* Disconnected from replica but allow to serve data */ + return C_OK; + } + + if (c->cmd->flags & CMD_STALE) { + /* Command is allow while stale */ + return C_OK; + } + + /* On stale replica, can not run the command */ + *err = sdsnew("Can not execute the command on a stale replica"); + return C_ERR; +} + /* Call a Redis command. * The reply is written to the run_ctx client and it is * up to the engine to take and parse. @@ -379,6 +418,10 @@ void scriptCall(scriptRunCtx *run_ctx, robj* *argv, int argc, sds *err) { return; } + if (scriptVerifyAllowStale(c, err) != C_OK) { + return; + } + if (scriptVerifyACL(c, err) != C_OK) { return; } diff --git a/src/script.h b/src/script.h index b2e253ee0..122db1dcf 100644 --- a/src/script.h +++ b/src/script.h @@ -62,6 +62,7 @@ #define SCRIPT_TIMEDOUT (1ULL<<3) /* indicate that the current script timedout */ #define SCRIPT_KILLED (1ULL<<4) /* indicate that the current script was marked to be killed */ #define SCRIPT_READ_ONLY (1ULL<<5) /* indicate that the current script should only perform read commands */ +#define SCRIPT_ALLOW_OOM (1ULL<<6) /* indicate to allow any command even if OOM reached */ #define SCRIPT_EVAL_MODE (1ULL<<7) /* Indicate that the current script called from legacy Lua */ typedef struct scriptRunCtx scriptRunCtx; @@ -75,6 +76,20 @@ struct scriptRunCtx { mstime_t snapshot_time; }; +/* Scripts flags */ +#define SCRIPT_FLAG_NO_WRITES (1ULL<<0) +#define SCRIPT_FLAG_ALLOW_OOM (1ULL<<1) +#define SCRIPT_FLAG_ALLOW_STALE (1ULL<<3) +#define SCRIPT_FLAG_NO_CLUSTER (1ULL<<4) + +/* Defines a script flags */ +typedef struct scriptFlag { + uint64_t flag; + const char *str; +} scriptFlag; + +extern scriptFlag scripts_flags_def[]; + void scriptPrepareForRun(scriptRunCtx *r_ctx, client *engine_client, client *caller, const char *funcname); void scriptResetRun(scriptRunCtx *r_ctx); int scriptSetResp(scriptRunCtx *r_ctx, int resp); diff --git a/src/script_lua.c b/src/script_lua.c index fc9fc812a..76e9eb7ad 100644 --- a/src/script_lua.c +++ b/src/script_lua.c @@ -35,6 +35,7 @@ #include "cluster.h" #include "monotonic.h" #include "resp_parser.h" +#include "version.h" #include #include #include @@ -1039,6 +1040,19 @@ static void luaRemoveUnsupportedFunctions(lua_State *lua) { lua_setglobal(lua,"dofile"); } +/* Return sds of the string value located on stack at the given index. + * Return NULL if the value is not a string. */ +sds luaGetStringSds(lua_State *lua, int index) { + if (!lua_isstring(lua, index)) { + return NULL; + } + + size_t len; + const char *str = lua_tolstring(lua, index, &len); + sds str_sds = sdsnewlen(str, len); + return str_sds; +} + /* This function installs metamethods in the global table _G that prevent * the creation of globals accidentally. * @@ -1142,6 +1156,16 @@ void luaSetGlobalProtection(lua_State *lua) { serverAssert(res == 0); } +void luaRegisterVersion(lua_State* lua) { + lua_pushstring(lua,"REDIS_VERSION_NUM"); + lua_pushnumber(lua,REDIS_VERSION_NUM); + lua_settable(lua,-3); + + lua_pushstring(lua,"REDIS_VERSION"); + lua_pushstring(lua,REDIS_VERSION); + lua_settable(lua,-3); +} + void luaRegisterLogFunction(lua_State* lua) { /* redis.log and log levels. */ lua_pushstring(lua,"log"); @@ -1184,6 +1208,8 @@ void luaRegisterRedisAPI(lua_State* lua) { luaRegisterLogFunction(lua); + luaRegisterVersion(lua); + /* redis.setresp */ lua_pushstring(lua,"setresp"); lua_pushcfunction(lua,luaSetResp); diff --git a/src/script_lua.h b/src/script_lua.h index b7862bee1..ac13178ca 100644 --- a/src/script_lua.h +++ b/src/script_lua.h @@ -59,10 +59,12 @@ #define REDIS_API_NAME "redis" void luaRegisterRedisAPI(lua_State* lua); +sds luaGetStringSds(lua_State *lua, int index); void luaEnableGlobalsProtection(lua_State *lua, int is_eval); void luaRegisterGlobalProtectionFunction(lua_State *lua); void luaSetGlobalProtection(lua_State *lua); void luaRegisterLogFunction(lua_State* lua); +void luaRegisterVersion(lua_State* lua); void luaPushError(lua_State *lua, char *error); int luaRaiseError(lua_State *lua); void luaSaveOnRegistry(lua_State* lua, const char* name, void* ptr); diff --git a/src/server.c b/src/server.c index 36143caa5..300d50198 100644 --- a/src/server.c +++ b/src/server.c @@ -3543,7 +3543,8 @@ int processCommand(client *c) { * arguments might interfere. */ if (c->cmd->proc == evalCommand || c->cmd->proc == evalShaCommand || - c->cmd->proc == fcallCommand) + c->cmd->proc == fcallCommand || + c->cmd->proc == fcallroCommand) { server.script_oom = out_of_memory; } diff --git a/tests/cluster/tests/00-base.tcl b/tests/cluster/tests/00-base.tcl index e731bf6a3..6ff043497 100644 --- a/tests/cluster/tests/00-base.tcl +++ b/tests/cluster/tests/00-base.tcl @@ -62,3 +62,9 @@ test "Sanity for CLUSTER COUNTKEYSINSLOT" { test "It is possible to write and read from the cluster" { cluster_write_test 0 } + +test "Function no-cluster flag" { + R 1 function load lua test {redis.register_function('f1', function() return 'hello' end, {'no-cluster'})} + catch {R 1 fcall f1 0} e + assert_match {*Can not run function on cluster, 'no-cluster' flag is set*} $e +} diff --git a/tests/integration/redis-cli.tcl b/tests/integration/redis-cli.tcl index 9cbec3f96..88b189ac0 100644 --- a/tests/integration/redis-cli.tcl +++ b/tests/integration/redis-cli.tcl @@ -338,7 +338,7 @@ if {!$::tls} { ;# fake_redis_node doesn't support TLS assert_equal "OK" [r function load lua should_not_exist_func "redis.register_function('should_not_exist_func', function() return 456 end)"] assert_equal "OK" [r debug reload nosave] assert_equal {} [r get should-not-exist] - assert_equal {{library_name lib1 engine LUA description {} functions {{name func1 description {}}}}} [r function list] + assert_equal {{library_name lib1 engine LUA description {} functions {{name func1 description {} flags {}}}}} [r function list] if {$functions_only} { assert_equal 0 [r dbsize] } else { diff --git a/tests/unit/cluster.tcl b/tests/unit/cluster.tcl index d1c02a7c5..99925688c 100644 --- a/tests/unit/cluster.tcl +++ b/tests/unit/cluster.tcl @@ -199,13 +199,13 @@ start_server [list overrides $base_conf] { } # make sure 'test' function was added to the new node - assert_equal {{library_name TEST engine LUA description {} functions {{name test description {}}}}} [$node4_rd FUNCTION LIST] + assert_equal {{library_name TEST engine LUA description {} functions {{name test description {} flags {}}}}} [$node4_rd FUNCTION LIST] # add function to node 5 assert_equal {OK} [$node5_rd FUNCTION LOAD LUA TEST {redis.register_function('test', function() return 'hello' end)}] # make sure functions was added to node 5 - assert_equal {{library_name TEST engine LUA description {} functions {{name test description {}}}}} [$node5_rd FUNCTION LIST] + assert_equal {{library_name TEST engine LUA description {} functions {{name test description {} flags {}}}}} [$node5_rd FUNCTION LIST] # adding node 5 to the cluster should failed because it already contains the 'test' function catch { diff --git a/tests/unit/functions.tcl b/tests/unit/functions.tcl index 84f9154e5..48f846e20 100644 --- a/tests/unit/functions.tcl +++ b/tests/unit/functions.tcl @@ -2,6 +2,10 @@ proc get_function_code {args} { return [format "redis.register_function('%s', function(KEYS, ARGV)\n %s \nend)" [lindex $args 0] [lindex $args 1]] } +proc get_no_writes_function_code {args} { + return [format "redis.register_function{function_name='%s', callback=function(KEYS, ARGV)\n %s \nend, flags={'no-writes'}}" [lindex $args 0] [lindex $args 1]] +} + start_server {tags {"scripting"}} { test {FUNCTION - Basic usage} { r function load LUA test [get_function_code test {return 'hello'}] @@ -68,7 +72,7 @@ start_server {tags {"scripting"}} { test {FUNCTION - test description argument} { r function load LUA test DESCRIPTION {some description} [get_function_code test {return 'hello'}] r function list - } {{library_name test engine LUA description {some description} functions {{name test description {}}}}} + } {{library_name test engine LUA description {some description} functions {{name test description {} flags {}}}}} test {FUNCTION - test fcall bad arguments} { catch { @@ -122,7 +126,7 @@ start_server {tags {"scripting"}} { assert_match "*Error trying to load the RDB*" $e r debug reload noflush merge r function list - } {{library_name test engine LUA description {some description} functions {{name test description {}}}}} {needs:debug} + } {{library_name test engine LUA description {some description} functions {{name test description {} flags {}}}}} {needs:debug} test {FUNCTION - test debug reload with nosave and noflush} { r function delete test @@ -141,7 +145,7 @@ start_server {tags {"scripting"}} { r flushall r flushdb r function list - } {{library_name test engine LUA description {} functions {{name test description {}}}}} + } {{library_name test engine LUA description {} functions {{name test description {} flags {}}}}} test {FUNCTION - test function dump and restore} { r function flush @@ -151,7 +155,7 @@ start_server {tags {"scripting"}} { assert_match {} [r function list] r function restore $e r function list - } {{library_name test engine LUA description {some description} functions {{name test description {}}}}} + } {{library_name test engine LUA description {some description} functions {{name test description {} flags {}}}}} test {FUNCTION - test function dump and restore with flush argument} { set e [r function dump] @@ -159,7 +163,7 @@ start_server {tags {"scripting"}} { assert_match {} [r function list] r function restore $e FLUSH r function list - } {{library_name test engine LUA description {some description} functions {{name test description {}}}}} + } {{library_name test engine LUA description {some description} functions {{name test description {} flags {}}}}} test {FUNCTION - test function dump and restore with append argument} { set e [r function dump] @@ -193,7 +197,7 @@ start_server {tags {"scripting"}} { catch {r function restore bad_payload} e assert_match {*payload version or checksum are wrong*} $e r function list - } {{library_name test engine LUA description {some description} functions {{name test description {}}}}} + } {{library_name test engine LUA description {some description} functions {{name test description {} flags {}}}}} test {FUNCTION - test function restore with wrong number of arguments} { catch {r function restore arg1 args2 arg3} e @@ -201,13 +205,13 @@ start_server {tags {"scripting"}} { } {*wrong number of arguments*} test {FUNCTION - test fcall_ro with write command} { - r function load lua test REPLACE [get_function_code test {return redis.call('set', 'x', '1')}] + r function load lua test REPLACE [get_no_writes_function_code test {return redis.call('set', 'x', '1')}] catch { r fcall_ro test 0 } e set _ $e } {*Write commands are not allowed from read-only scripts*} test {FUNCTION - test fcall_ro with read only commands} { - r function load lua test REPLACE [get_function_code test {return redis.call('get', 'x')}] + r function load lua test REPLACE [get_no_writes_function_code test {return redis.call('get', 'x')}] r set x 1 r fcall_ro test 0 } {1} @@ -271,17 +275,17 @@ start_server {tags {"scripting"}} { test {FUNCTION - test function flush} { r function load lua test REPLACE [get_function_code test {local a = 1 while true do a = a + 1 end}] - assert_match {{library_name test engine LUA description {} functions {{name test description {}}}}} [r function list] + assert_match {{library_name test engine LUA description {} functions {{name test description {} flags {}}}}} [r function list] r function flush assert_match {} [r function list] r function load lua test REPLACE [get_function_code test {local a = 1 while true do a = a + 1 end}] - assert_match {{library_name test engine LUA description {} functions {{name test description {}}}}} [r function list] + assert_match {{library_name test engine LUA description {} functions {{name test description {} flags {}}}}} [r function list] r function flush async assert_match {} [r function list] r function load lua test REPLACE [get_function_code test {local a = 1 while true do a = a + 1 end}] - assert_match {{library_name test engine LUA description {} functions {{name test description {}}}}} [r function list] + assert_match {{library_name test engine LUA description {} functions {{name test description {} flags {}}}}} [r function list] r function flush sync assert_match {} [r function list] } @@ -308,9 +312,9 @@ start_server {tags {"scripting repl external:skip"}} { } test {FUNCTION - creation is replicated to replica} { - r function load LUA test DESCRIPTION {some description} [get_function_code test {return 'hello'}] - wait_for_condition 50 100 { - [r -1 function list] eq {{library_name test engine LUA description {some description} functions {{name test description {}}}}} + r function load LUA test DESCRIPTION {some description} [get_no_writes_function_code test {return 'hello'}] + wait_for_condition 50 100 { + [r -1 function list] eq {{library_name test engine LUA description {some description} functions {{name test description {} flags no-writes}}}} } else { fail "Failed waiting for function to replicate to replica" } @@ -333,7 +337,7 @@ start_server {tags {"scripting repl external:skip"}} { assert_equal [r function restore $e] {OK} wait_for_condition 50 100 { - [r -1 function list] eq {{library_name test engine LUA description {some description} functions {{name test description {}}}}} + [r -1 function list] eq {{library_name test engine LUA description {some description} functions {{name test description {} flags no-writes}}}} } else { fail "Failed waiting for function to replicate to replica" } @@ -351,7 +355,7 @@ start_server {tags {"scripting repl external:skip"}} { test {FUNCTION - flush is replicated to replica} { r function load LUA test DESCRIPTION {some description} [get_function_code test {return 'hello'}] wait_for_condition 50 100 { - [r -1 function list] eq {{library_name test engine LUA description {some description} functions {{name test description {}}}}} + [r -1 function list] eq {{library_name test engine LUA description {some description} functions {{name test description {} flags {}}}}} } else { fail "Failed waiting for function to replicate to replica" } @@ -367,7 +371,7 @@ start_server {tags {"scripting repl external:skip"}} { r -1 slaveof no one # creating a function after disconnect to make sure function # is replicated on rdb phase - r function load LUA test DESCRIPTION {some description} [get_function_code test {return 'hello'}] + r function load LUA test DESCRIPTION {some description} [get_no_writes_function_code test {return 'hello'}] # reconnect the replica r -1 slaveof [srv 0 host] [srv 0 port] @@ -385,7 +389,7 @@ start_server {tags {"scripting repl external:skip"}} { test "FUNCTION - test replication to replica on rdb phase info command" { r -1 function list - } {{library_name test engine LUA description {some description} functions {{name test description {}}}}} + } {{library_name test engine LUA description {some description} functions {{name test description {} flags no-writes}}}} test "FUNCTION - create on read only replica" { catch { @@ -417,7 +421,7 @@ start_server {tags {"scripting repl external:skip"}} { r -1 fcall test 0 } e set _ $e - } {*can't write against a read only replica*} + } {*Can not run a function with write flag on readonly replica*} } } @@ -430,7 +434,7 @@ test {FUNCTION can processes create, delete and flush commands in AOF when doing r slaveof 127.0.0.1 0 r debug loadaof r slaveof no one - assert_equal [r function list] {{library_name test engine LUA description {} functions {{name test description {}}}}} + assert_equal [r function list] {{library_name test engine LUA description {} functions {{name test description {} flags {}}}}} r FUNCTION DELETE test @@ -474,15 +478,13 @@ start_server {tags {"scripting"}} { 'f1', function(keys, args) return add1(1) - end, - 'f1 description' + end ) redis.register_function( 'f2', function(keys, args) return add1(2) - end, - 'f2 description' + end ) } assert_equal [r fcall f1 0] {2} @@ -565,12 +567,12 @@ start_server {tags {"scripting"}} { } } e set _ $e - } {*wrong number of arguments to redis.register_function*} + } {*calling redis.register_function with a single argument is only applicable to Lua table*} test {LIBRARIES - test registration with to many arguments} { catch { r function load LUA lib2 replace { - redis.register_function('f1', function() return 1 end, 'description', 'extra arg') + redis.register_function('f1', function() return 1 end, {}, 'description', 'extra arg') } } e set _ $e @@ -741,6 +743,104 @@ start_server {tags {"scripting"}} { set _ $e } {*attempted to create global variable 'a'*} + test {LIBRARIES - named arguments} { + r function load LUA lib { + redis.register_function{ + function_name='f1', + callback=function() + return 'hello' + end, + description='some desc' + } + } + r function list + } {{library_name lib engine LUA description {} functions {{name f1 description {some desc} flags {}}}}} + + test {LIBRARIES - named arguments, bad function name} { + catch { + r function load LUA lib replace { + redis.register_function{ + function_name=function() return 1 end, + callback=function() + return 'hello' + end, + description='some desc' + } + } + } e + set _ $e + } {*function_name argument given to redis.register_function must be a string*} + + test {LIBRARIES - named arguments, bad callback type} { + catch { + r function load LUA lib replace { + redis.register_function{ + function_name='f1', + callback='bad', + description='some desc' + } + } + } e + set _ $e + } {*callback argument given to redis.register_function must be a function*} + + test {LIBRARIES - named arguments, bad description} { + catch { + r function load LUA lib replace { + redis.register_function{ + function_name='f1', + callback=function() + return 'hello' + end, + description=function() return 1 end + } + } + } e + set _ $e + } {*description argument given to redis.register_function must be a string*} + + test {LIBRARIES - named arguments, unknown argument} { + catch { + r function load LUA lib replace { + redis.register_function{ + function_name='f1', + callback=function() + return 'hello' + end, + description='desc', + some_unknown='unknown' + } + } + } e + set _ $e + } {*unknown argument given to redis.register_function*} + + test {LIBRARIES - named arguments, missing function name} { + catch { + r function load LUA lib replace { + redis.register_function{ + callback=function() + return 'hello' + end, + description='desc' + } + } + } e + set _ $e + } {*redis.register_function must get a function name argument*} + + test {LIBRARIES - named arguments, missing callback} { + catch { + r function load LUA lib replace { + redis.register_function{ + function_name='f1', + description='desc' + } + } + } e + set _ $e + } {*redis.register_function must get a callback argument*} + test {FUNCTION - test function restore with function name collision} { r function flush r function load lua lib1 { @@ -821,12 +921,12 @@ start_server {tags {"scripting"}} { r function flush r function load lua library1 {redis.register_function('f6', function(keys, args) return 7 end)} r function list withcode - } {{library_name library1 engine LUA description {} functions {{name f6 description {}}} library_code {redis.register_function('f6', function(keys, args) return 7 end)}}} + } {{library_name library1 engine LUA description {} functions {{name f6 description {} flags {}}} library_code {redis.register_function('f6', function(keys, args) return 7 end)}}} test {FUNCTION - test function list with pattern} { r function load lua lib1 {redis.register_function('f7', function(keys, args) return 7 end)} r function list libraryname library* - } {{library_name library1 engine LUA description {} functions {{name f6 description {}}}}} + } {{library_name library1 engine LUA description {} functions {{name f6 description {} flags {}}}}} test {FUNCTION - test function list wrong argument} { catch {r function list bad_argument} e @@ -864,4 +964,135 @@ start_server {tags {"scripting"}} { r config set maxmemory 0 } + + test {FUNCTION - verify allow-omm allows running any command} { + r FUNCTION load lua f1 replace { redis.register_function{ + function_name='f1', + callback=function() return redis.call('set', 'x', '1') end, + flags={'allow-oom'} + }} + + r config set maxmemory 1 + + assert_match {OK} [r fcall f1 1 k] + assert_match {1} [r get x] + + r config set maxmemory 0 + } +} + +start_server {tags {"scripting"}} { + test {FUNCTION - wrong flags type named arguments} { + catch {r function load lua test replace {redis.register_function{ + function_name = 'f1', + callback = function() return 1 end, + flags = 'bad flags type' + }}} e + set _ $e + } {*flags argument to redis.register_function must be a table representing function flags*} + + test {FUNCTION - wrong flag type} { + catch {r function load lua test replace {redis.register_function{ + function_name = 'f1', + callback = function() return 1 end, + flags = {function() return 1 end} + }}} e + set _ $e + } {*unknown flag given*} + + test {FUNCTION - unknown flag} { + catch {r function load lua test replace {redis.register_function{ + function_name = 'f1', + callback = function() return 1 end, + flags = {'unknown'} + }}} e + set _ $e + } {*unknown flag given*} + + test {FUNCTION - write script on fcall_ro} { + r function load lua test replace {redis.register_function{ + function_name = 'f1', + callback = function() return redis.call('set', 'x', 1) end + }} + catch {r fcall_ro f1 0} e + set _ $e + } {*Can not execute a function with write flag using fcall_ro*} + + test {FUNCTION - write script with no-writes flag} { + r function load lua test replace {redis.register_function{ + function_name = 'f1', + callback = function() return redis.call('set', 'x', 1) end, + flags = {'no-writes'} + }} + catch {r fcall f1 0} e + set _ $e + } {*Write commands are not allowed from read-only scripts*} + + test {FUNCTION - deny oom} { + r FUNCTION load lua test replace { redis.register_function('f1', function() return redis.call('set', 'x', '1') end) } + + r config set maxmemory 1 + + catch {[r fcall f1 1 k]} e + assert_match {*can not run it when used memory > 'maxmemory'*} $e + + r config set maxmemory 0 + } + + test {FUNCTION - deny oom on no-writes function} { + r FUNCTION load lua test replace {redis.register_function{function_name='f1', callback=function() return 'hello' end, flags={'no-writes'}}} + + r config set maxmemory 1 + + catch {r fcall f1 1 k} e + assert_match {*can not run it when used memory > 'maxmemory'*} $e + + catch {r fcall_ro f1 1 k} e + assert_match {*can not run it when used memory > 'maxmemory'*} $e + + r config set maxmemory 0 + } + + test {FUNCTION - allow stale} { + r FUNCTION load lua test replace { + redis.register_function{function_name='f1', callback=function() return 'hello' end, flags={'no-writes'}} + redis.register_function{function_name='f2', callback=function() return 'hello' end, flags={'allow-stale', 'no-writes'}} + redis.register_function{function_name='f3', callback=function() return redis.call('get', 'x') end, flags={'allow-stale', 'no-writes'}} + redis.register_function{function_name='f4', callback=function() return redis.call('info', 'server') end, flags={'allow-stale', 'no-writes'}} + } + + r config set replica-serve-stale-data no + r replicaof 127.0.0.1 1 + + catch {[r fcall f1 0]} e + assert_match {*'allow-stale' flag is not set on the function*} $e + + assert_equal {hello} [r fcall f2 0] + + catch {[r fcall f3 0]} e + assert_match {*Can not execute the command on a stale replica*} $e + + assert_match {*redis_version*} [r fcall f4 0] + + r replicaof no one + r config set replica-serve-stale-data yes + set _ {} + } {} {external:skip} + + test {FUNCTION - redis version api} { + r FUNCTION load lua test replace { + local version = redis.REDIS_VERSION_NUM + + redis.register_function{function_name='get_version_v1', callback=function() + return string.format('%s.%s.%s', + bit.band(bit.rshift(version, 4), 0x000000ff), + bit.band(bit.rshift(version, 2), 0x000000ff), + bit.band(version, 0x000000ff)) + end} + redis.register_function{function_name='get_version_v2', callback=function() return redis.REDIS_VERSION end} + } + + catch {[r fcall f1 0]} e + assert_equal [r fcall get_version_v1 0] [r fcall get_version_v2 0] + } } diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index 96511bad5..312557283 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -19,7 +19,7 @@ if {$is_eval == 1} { r fcall test {*}[lrange $args 1 end] } proc run_script_ro {args} { - r function load LUA test replace [format "redis.register_function('test', function(KEYS, ARGV)\n %s \nend)" [lindex $args 0]] + r function load LUA test replace [format "redis.register_function{function_name='test', callback=function(KEYS, ARGV)\n %s \nend, flags={'no-writes'}}" [lindex $args 0]] r fcall_ro test {*}[lrange $args 1 end] } proc run_script_on_connection {args} { @@ -36,19 +36,16 @@ if {$is_eval == 1} { start_server {tags {"scripting"}} { + if {$is_eval eq 1} { test {Script - disallow write on OOM} { - r FUNCTION load lua f1 replace { redis.register_function('f1', function() return redis.call('set', 'x', '1') end) } - r config set maxmemory 1 - catch {[r fcall f1 1 k]} e - assert_match {*command not allowed when used memory*} $e - catch {[r eval "redis.call('set', 'x', 1)" 0]} e assert_match {*command not allowed when used memory*} $e r config set maxmemory 0 } + } ;# is_eval test {EVAL - Does Lua interpreter replies to our requests?} { run_script {return 'hello'} 0