Fix Lua C API violation on lua msgpack lib. (#9832)

msgpack lib missed using lua_checkstack and so on rare
cases overflow the stack by at most 2 elements. This is a
violation of the Lua C API. Notice that Lua allocates
additional 5 more elements on top of lua->stack_last
so Redis does not access an invalid memory. But it is an
API violation and we should avoid it.

This PR also added a new Lua compilation option. The new
option can be enable using environment variable called
LUA_DEBUG. If set to `yes` (by default `no`), Lua will be
compiled without optimizations and with debug symbols (`-O0 -g`).
In addition, in this new mode, Lua will be compiled with the
`-DLUA_USE_APICHECK` flag that enables extended Lua C API
validations.

In addition, set LUA_DEBUG=yes on daily valgrind flow so we
will be able to catch Lua C API violations in the future.

.github/workflows/daily.yml

@@ -370,7 +370,7 @@ jobs:
           repository: ${{ env.GITHUB_REPOSITORY }}
           ref: ${{ env.GITHUB_HEAD_REF }}
       - name: make
-        run: make SANITIZER=undefined REDIS_CFLAGS='-DREDIS_TEST'
+        run: make SANITIZER=undefined REDIS_CFLAGS='-DREDIS_TEST' LUA_DEBUG=yes # we (ab)use this flow to also check Lua C API violations
       - name: testprep
         run: |
           sudo apt-get update

deps/Makefile

@@ -2,6 +2,8 @@
 
 uname_S:= $(shell sh -c 'uname -s 2>/dev/null || echo not')
 
+LUA_DEBUG?=no
+
 CCCOLOR="\033[34m"
 LINKCOLOR="\033[34;1m"
 SRCCOLOR="\033[33m"
@@ -69,8 +71,13 @@ ifeq ($(uname_S),SunOS)
 	LUA_CFLAGS= -D__C99FEATURES__=1
 endif
 
-LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' -DLUA_USE_MKSTEMP $(CFLAGS)
+LUA_CFLAGS+= -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' -DLUA_USE_MKSTEMP $(CFLAGS)
 LUA_LDFLAGS+= $(LDFLAGS)
+ifeq ($(LUA_DEBUG),yes)
+	LUA_CFLAGS+= -O0 -g -DLUA_USE_APICHECK
+else
+	LUA_CFLAGS+= -O2 
+endif
 # lua's Makefile defines AR="ar rcu", which is unusual, and makes it more
 # challenging to cross-compile lua (and redis).  These defines make it easier
 # to fit redis into cross-compilation environments, which typically set AR.

deps/lua/src/lua_cmsgpack.c

@@ -435,6 +435,7 @@ int table_is_an_array(lua_State *L) {
 
     stacktop = lua_gettop(L);
 
+    luaL_checkstack(L, 2, "in function table_is_an_array");
     lua_pushnil(L);
     while(lua_next(L,-2)) {
         /* Stack: ... key value */