Initial RocksDB integration

Former-commit-id: 0de9e5b692c02e779e538ddd0a56d10215e501bb

deps/Makefile

@@ -83,13 +83,13 @@ JEMALLOC_LDFLAGS= $(LDFLAGS)
 
 jemalloc: .make-prerequisites
 	@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
-	cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --enable-cc-silence CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)"
+	cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --enable-cc-silence --disable-cxx CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)"
 	cd jemalloc && $(MAKE) CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" lib/libjemalloc.a
 
 .PHONY: jemalloc
 
 rocksdb: .make-prerequisites
 	@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
-	cd rocksdb && $(MAKE) shared_lib
+	cd rocksdb && $(MAKE) static_lib
 
 .PHONY: rocksdb

src/IStorage.h

@@ -10,7 +10,7 @@ public:
 
     virtual void insert(const char *key, size_t cchKey, void *data, size_t cb) = 0;
     virtual void erase(const char *key, size_t cchKey) = 0;
-    virtual void retrieve(const char *key, size_t cchKey, bool fDelete, callback fn) const = 0;
+    virtual void retrieve(const char *key, size_t cchKey, callback fn) const = 0;
     virtual size_t clear() = 0;
     virtual void enumerate(callback fn) const = 0;
 

src/Makefile

@@ -103,7 +103,7 @@ endif
 FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS)
 FINAL_CXXFLAGS=$(CXX_STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(CXXFLAGS) $(REDIS_CFLAGS)
 FINAL_LDFLAGS=$(LDFLAGS) $(REDIS_LDFLAGS) $(DEBUG)
-FINAL_LIBS=-lm
+FINAL_LIBS=-lm -lz
 DEBUG=-g -ggdb
 
 ifeq ($(uname_S),SunOS)
@@ -258,8 +258,8 @@ endif
 	@touch $@
 
 # keydb-server
-$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) librocksdb.so
-	$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a librocksdb.so -Wl,-rpath . $(FINAL_LIBS)
+$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) 
+	$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/rocksdb/librocksdb.a $(FINAL_LIBS)
 
 # keydb-sentinel
 $(REDIS_SENTINEL_NAME): $(REDIS_SERVER_NAME)
@@ -296,12 +296,8 @@ dict-benchmark: dict.cpp zmalloc.cpp sds.c siphash.c
 %.o: %.asm .make-prerequisites
 	$(KEYDB_AS) $< -o $@
 
-librocksdb.so: .make-prerequisites
-	$(QUIET_CP)cp --preserve=mode ../deps/rocksdb/librocksdb.so* ./
-
 clean:
 	rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep dict-benchmark
-	rm -rf librocksdb.so*
 
 .PHONY: clean
 

src/db.cpp

@@ -1903,6 +1903,17 @@ void redisDbPersistentData::initialize()
     m_fTrackingChanges = 0;
 }
 
+void redisDbPersistentData::setStorageProvider(IStorage *pstorage)
+{
+    serverAssert(m_spstorage == nullptr);
+    m_spstorage = std::unique_ptr<IStorage>(pstorage);
+    m_spstorage->enumerate([&](const char *key, size_t cchkey, const void *, size_t){
+        sds sdsKey = sdsnewlen(key, cchkey);
+        dictAdd(m_pdict,  sdsKey,  nullptr);
+    });
+}
+
+IStorage *create_rocksdb_storage(const char *dbfile);
 void redisDb::initialize(int id)
 {
     redisDbPersistentData::initialize();
@@ -1914,6 +1925,8 @@ void redisDb::initialize(int id)
     this->avg_ttl = 0;
     this->last_expire_set = 0;
     this->defrag_later = listCreate();
+    if (id == 0)
+        this->setStorageProvider(create_rocksdb_storage("/tmp/rocks.db"));
 }
 
 bool redisDbPersistentData::insert(char *key, robj *o)
@@ -2068,7 +2081,7 @@ void redisDbPersistentData::ensure(const char *sdsKey, dictEntry **pde)
     else if (*pde != nullptr && dictGetVal(*pde) == nullptr)
     {
         serverAssert(m_spstorage != nullptr);
-        m_spstorage->retrieve(sdsKey, sdslen(sdsKey), true, [&](const char *, size_t, const void *data, size_t cb){
+        m_spstorage->retrieve(sdsKey, sdslen(sdsKey), [&](const char *, size_t, const void *data, size_t cb){
             robj *o = deserializeStoredObject(data, cb);
             serverAssert(o != nullptr);
             dictSetVal(m_pdict, *pde, o);

src/new.cpp

@@ -32,3 +32,8 @@ void operator delete(void *p, std::size_t) noexcept
 }
 
 #endif
+
+extern "C" size_t malloc_usable_size(void *pv)
+{
+    return zmalloc_size(pv);
+}

src/new.h

@@ -1,5 +1,6 @@
 #pragma once
 #include <cstddef>  // std::size_t
+#include "storage.h"
 
 void *operator new(size_t size, enum MALLOC_CLASS mclass);
 

src/server.cpp

@@ -5057,6 +5057,27 @@ void OnTerminate()
         The easiest way to achieve that is to acutally segfault, so we assert
         here.
     */
+    auto exception = std::current_exception();
+    if (exception != nullptr)
+    {
+        try
+        {
+            std::rethrow_exception(exception);
+        }
+        catch (const char *szErr)
+        {
+            serverLog(LL_WARNING, "Crashing on uncaught exception: %s", szErr);
+        }
+        catch (std::string str)
+        {
+            serverLog(LL_WARNING, "Crashing on uncaught exception: %s", str.c_str());
+        }
+        catch (...)
+        {
+            // NOP
+        }
+    }
+
     serverAssert(false);
 }
 

src/server.h

@@ -1187,6 +1187,7 @@ public:
     robj *val() { return de ? (robj*)dictGetVal(de) : nullptr; }
     robj *operator->() { return de ? (robj*)dictGetVal(de) : nullptr; }
     operator robj*() const { return de ? (robj*)dictGetVal(de) : nullptr; }
+
 };
 
 class redisDbPersistentDataSnapshot;
@@ -1266,6 +1267,8 @@ public:
     const expireEntry *getExpire(robj_roptr key) const;
     void initialize();
 
+    void setStorageProvider(IStorage *pstorage);
+
     void trackChanges() { m_fTrackingChanges++; }
     void processChanges();
 
@@ -1406,6 +1409,16 @@ typedef struct redisDb : public redisDbPersistentDataSnapshot
     using redisDbPersistentData::endSnapshot;
     using redisDbPersistentData::consolidate_snapshot;
 
+    dict_iter find_threadsafe(const char *key) const
+    {
+        dict_iter itr = redisDbPersistentDataSnapshot::find_threadsafe(key);
+        if (itr.key() != nullptr && itr.val() == nullptr)
+        {
+            return const_cast<redisDb*>(this)->redisDbPersistentData::find(key);
+        }
+        return itr;
+    }
+
 public:
     expireset::setiter expireitr;
     dict *blocking_keys;        /* Keys with clients waiting for data (BLPOP)*/