From a8c1253b6fd1d85ba33a8749e14e4fa515508df0 Mon Sep 17 00:00:00 2001
From: "Meir Shpilraien (Spielrein)" <meir@redis.com>
Date: Sun, 28 Nov 2021 11:33:09 +0200
Subject: [PATCH] Fix Lua C API violation on lua msgpack lib. (#9832)

msgpack lib missed using lua_checkstack and so on rare
cases overflow the stack by at most 2 elements. This is a
violation of the Lua C API. Notice that Lua allocates
additional 5 more elements on top of lua->stack_last
so Redis does not access an invalid memory. But it is an
API violation and we should avoid it.

This PR also added a new Lua compilation option. The new
option can be enable using environment variable called
LUA_DEBUG. If set to `yes` (by default `no`), Lua will be
compiled without optimizations and with debug symbols (`-O0 -g`).
In addition, in this new mode, Lua will be compiled with the
`-DLUA_USE_APICHECK` flag that enables extended Lua C API
validations.

In addition, set LUA_DEBUG=yes on daily valgrind flow so we
will be able to catch Lua C API violations in the future.
---
 .github/workflows/daily.yml | 2 +-
 deps/Makefile               | 9 ++++++++-
 deps/lua/src/lua_cmsgpack.c | 1 +
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index faaaf02ef..a3e31e01a 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -370,7 +370,7 @@ jobs:
           repository: ${{ env.GITHUB_REPOSITORY }}
           ref: ${{ env.GITHUB_HEAD_REF }}
       - name: make
-        run: make SANITIZER=undefined REDIS_CFLAGS='-DREDIS_TEST'
+        run: make SANITIZER=undefined REDIS_CFLAGS='-DREDIS_TEST' LUA_DEBUG=yes # we (ab)use this flow to also check Lua C API violations
       - name: testprep
         run: |
           sudo apt-get update
diff --git a/deps/Makefile b/deps/Makefile
index 00fdbaf7c..8592e1766 100644
--- a/deps/Makefile
+++ b/deps/Makefile
@@ -2,6 +2,8 @@
 
 uname_S:= $(shell sh -c 'uname -s 2>/dev/null || echo not')
 
+LUA_DEBUG?=no
+
 CCCOLOR="\033[34m"
 LINKCOLOR="\033[34;1m"
 SRCCOLOR="\033[33m"
@@ -69,8 +71,13 @@ ifeq ($(uname_S),SunOS)
 	LUA_CFLAGS= -D__C99FEATURES__=1
 endif
 
-LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' -DLUA_USE_MKSTEMP $(CFLAGS)
+LUA_CFLAGS+= -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' -DLUA_USE_MKSTEMP $(CFLAGS)
 LUA_LDFLAGS+= $(LDFLAGS)
+ifeq ($(LUA_DEBUG),yes)
+	LUA_CFLAGS+= -O0 -g -DLUA_USE_APICHECK
+else
+	LUA_CFLAGS+= -O2 
+endif
 # lua's Makefile defines AR="ar rcu", which is unusual, and makes it more
 # challenging to cross-compile lua (and redis).  These defines make it easier
 # to fit redis into cross-compilation environments, which typically set AR.
diff --git a/deps/lua/src/lua_cmsgpack.c b/deps/lua/src/lua_cmsgpack.c
index 892154793..49c6dc7b0 100644
--- a/deps/lua/src/lua_cmsgpack.c
+++ b/deps/lua/src/lua_cmsgpack.c
@@ -435,6 +435,7 @@ int table_is_an_array(lua_State *L) {
 
     stacktop = lua_gettop(L);
 
+    luaL_checkstack(L, 2, "in function table_is_an_array");
     lua_pushnil(L);
     while(lua_next(L,-2)) {
         /* Stack: ... key value */