Initial RocksDB integration
Former-commit-id: 0de9e5b692c02e779e538ddd0a56d10215e501bb
This commit is contained in:
parent
11f710c5fc
commit
130589d8d1
4
deps/Makefile
vendored
4
deps/Makefile
vendored
@ -83,13 +83,13 @@ JEMALLOC_LDFLAGS= $(LDFLAGS)
|
|||||||
|
|
||||||
jemalloc: .make-prerequisites
|
jemalloc: .make-prerequisites
|
||||||
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
|
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
|
||||||
cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --enable-cc-silence CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)"
|
cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --enable-cc-silence --disable-cxx CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)"
|
||||||
cd jemalloc && $(MAKE) CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" lib/libjemalloc.a
|
cd jemalloc && $(MAKE) CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" lib/libjemalloc.a
|
||||||
|
|
||||||
.PHONY: jemalloc
|
.PHONY: jemalloc
|
||||||
|
|
||||||
rocksdb: .make-prerequisites
|
rocksdb: .make-prerequisites
|
||||||
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
|
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
|
||||||
cd rocksdb && $(MAKE) shared_lib
|
cd rocksdb && $(MAKE) static_lib
|
||||||
|
|
||||||
.PHONY: rocksdb
|
.PHONY: rocksdb
|
||||||
|
@ -10,7 +10,7 @@ public:
|
|||||||
|
|
||||||
virtual void insert(const char *key, size_t cchKey, void *data, size_t cb) = 0;
|
virtual void insert(const char *key, size_t cchKey, void *data, size_t cb) = 0;
|
||||||
virtual void erase(const char *key, size_t cchKey) = 0;
|
virtual void erase(const char *key, size_t cchKey) = 0;
|
||||||
virtual void retrieve(const char *key, size_t cchKey, bool fDelete, callback fn) const = 0;
|
virtual void retrieve(const char *key, size_t cchKey, callback fn) const = 0;
|
||||||
virtual size_t clear() = 0;
|
virtual size_t clear() = 0;
|
||||||
virtual void enumerate(callback fn) const = 0;
|
virtual void enumerate(callback fn) const = 0;
|
||||||
|
|
||||||
|
10
src/Makefile
10
src/Makefile
@ -103,7 +103,7 @@ endif
|
|||||||
FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS)
|
FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS)
|
||||||
FINAL_CXXFLAGS=$(CXX_STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(CXXFLAGS) $(REDIS_CFLAGS)
|
FINAL_CXXFLAGS=$(CXX_STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(CXXFLAGS) $(REDIS_CFLAGS)
|
||||||
FINAL_LDFLAGS=$(LDFLAGS) $(REDIS_LDFLAGS) $(DEBUG)
|
FINAL_LDFLAGS=$(LDFLAGS) $(REDIS_LDFLAGS) $(DEBUG)
|
||||||
FINAL_LIBS=-lm
|
FINAL_LIBS=-lm -lz
|
||||||
DEBUG=-g -ggdb
|
DEBUG=-g -ggdb
|
||||||
|
|
||||||
ifeq ($(uname_S),SunOS)
|
ifeq ($(uname_S),SunOS)
|
||||||
@ -258,8 +258,8 @@ endif
|
|||||||
@touch $@
|
@touch $@
|
||||||
|
|
||||||
# keydb-server
|
# keydb-server
|
||||||
$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) librocksdb.so
|
$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ)
|
||||||
$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a librocksdb.so -Wl,-rpath . $(FINAL_LIBS)
|
$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/rocksdb/librocksdb.a $(FINAL_LIBS)
|
||||||
|
|
||||||
# keydb-sentinel
|
# keydb-sentinel
|
||||||
$(REDIS_SENTINEL_NAME): $(REDIS_SERVER_NAME)
|
$(REDIS_SENTINEL_NAME): $(REDIS_SERVER_NAME)
|
||||||
@ -296,12 +296,8 @@ dict-benchmark: dict.cpp zmalloc.cpp sds.c siphash.c
|
|||||||
%.o: %.asm .make-prerequisites
|
%.o: %.asm .make-prerequisites
|
||||||
$(KEYDB_AS) $< -o $@
|
$(KEYDB_AS) $< -o $@
|
||||||
|
|
||||||
librocksdb.so: .make-prerequisites
|
|
||||||
$(QUIET_CP)cp --preserve=mode ../deps/rocksdb/librocksdb.so* ./
|
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep dict-benchmark
|
rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep dict-benchmark
|
||||||
rm -rf librocksdb.so*
|
|
||||||
|
|
||||||
.PHONY: clean
|
.PHONY: clean
|
||||||
|
|
||||||
|
15
src/db.cpp
15
src/db.cpp
@ -1903,6 +1903,17 @@ void redisDbPersistentData::initialize()
|
|||||||
m_fTrackingChanges = 0;
|
m_fTrackingChanges = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void redisDbPersistentData::setStorageProvider(IStorage *pstorage)
|
||||||
|
{
|
||||||
|
serverAssert(m_spstorage == nullptr);
|
||||||
|
m_spstorage = std::unique_ptr<IStorage>(pstorage);
|
||||||
|
m_spstorage->enumerate([&](const char *key, size_t cchkey, const void *, size_t){
|
||||||
|
sds sdsKey = sdsnewlen(key, cchkey);
|
||||||
|
dictAdd(m_pdict, sdsKey, nullptr);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
IStorage *create_rocksdb_storage(const char *dbfile);
|
||||||
void redisDb::initialize(int id)
|
void redisDb::initialize(int id)
|
||||||
{
|
{
|
||||||
redisDbPersistentData::initialize();
|
redisDbPersistentData::initialize();
|
||||||
@ -1914,6 +1925,8 @@ void redisDb::initialize(int id)
|
|||||||
this->avg_ttl = 0;
|
this->avg_ttl = 0;
|
||||||
this->last_expire_set = 0;
|
this->last_expire_set = 0;
|
||||||
this->defrag_later = listCreate();
|
this->defrag_later = listCreate();
|
||||||
|
if (id == 0)
|
||||||
|
this->setStorageProvider(create_rocksdb_storage("/tmp/rocks.db"));
|
||||||
}
|
}
|
||||||
|
|
||||||
bool redisDbPersistentData::insert(char *key, robj *o)
|
bool redisDbPersistentData::insert(char *key, robj *o)
|
||||||
@ -2068,7 +2081,7 @@ void redisDbPersistentData::ensure(const char *sdsKey, dictEntry **pde)
|
|||||||
else if (*pde != nullptr && dictGetVal(*pde) == nullptr)
|
else if (*pde != nullptr && dictGetVal(*pde) == nullptr)
|
||||||
{
|
{
|
||||||
serverAssert(m_spstorage != nullptr);
|
serverAssert(m_spstorage != nullptr);
|
||||||
m_spstorage->retrieve(sdsKey, sdslen(sdsKey), true, [&](const char *, size_t, const void *data, size_t cb){
|
m_spstorage->retrieve(sdsKey, sdslen(sdsKey), [&](const char *, size_t, const void *data, size_t cb){
|
||||||
robj *o = deserializeStoredObject(data, cb);
|
robj *o = deserializeStoredObject(data, cb);
|
||||||
serverAssert(o != nullptr);
|
serverAssert(o != nullptr);
|
||||||
dictSetVal(m_pdict, *pde, o);
|
dictSetVal(m_pdict, *pde, o);
|
||||||
|
@ -32,3 +32,8 @@ void operator delete(void *p, std::size_t) noexcept
|
|||||||
}
|
}
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
extern "C" size_t malloc_usable_size(void *pv)
|
||||||
|
{
|
||||||
|
return zmalloc_size(pv);
|
||||||
|
}
|
@ -1,5 +1,6 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
#include <cstddef> // std::size_t
|
#include <cstddef> // std::size_t
|
||||||
|
#include "storage.h"
|
||||||
|
|
||||||
void *operator new(size_t size, enum MALLOC_CLASS mclass);
|
void *operator new(size_t size, enum MALLOC_CLASS mclass);
|
||||||
|
|
||||||
|
@ -5057,6 +5057,27 @@ void OnTerminate()
|
|||||||
The easiest way to achieve that is to acutally segfault, so we assert
|
The easiest way to achieve that is to acutally segfault, so we assert
|
||||||
here.
|
here.
|
||||||
*/
|
*/
|
||||||
|
auto exception = std::current_exception();
|
||||||
|
if (exception != nullptr)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
std::rethrow_exception(exception);
|
||||||
|
}
|
||||||
|
catch (const char *szErr)
|
||||||
|
{
|
||||||
|
serverLog(LL_WARNING, "Crashing on uncaught exception: %s", szErr);
|
||||||
|
}
|
||||||
|
catch (std::string str)
|
||||||
|
{
|
||||||
|
serverLog(LL_WARNING, "Crashing on uncaught exception: %s", str.c_str());
|
||||||
|
}
|
||||||
|
catch (...)
|
||||||
|
{
|
||||||
|
// NOP
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
serverAssert(false);
|
serverAssert(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
13
src/server.h
13
src/server.h
@ -1187,6 +1187,7 @@ public:
|
|||||||
robj *val() { return de ? (robj*)dictGetVal(de) : nullptr; }
|
robj *val() { return de ? (robj*)dictGetVal(de) : nullptr; }
|
||||||
robj *operator->() { return de ? (robj*)dictGetVal(de) : nullptr; }
|
robj *operator->() { return de ? (robj*)dictGetVal(de) : nullptr; }
|
||||||
operator robj*() const { return de ? (robj*)dictGetVal(de) : nullptr; }
|
operator robj*() const { return de ? (robj*)dictGetVal(de) : nullptr; }
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
class redisDbPersistentDataSnapshot;
|
class redisDbPersistentDataSnapshot;
|
||||||
@ -1266,6 +1267,8 @@ public:
|
|||||||
const expireEntry *getExpire(robj_roptr key) const;
|
const expireEntry *getExpire(robj_roptr key) const;
|
||||||
void initialize();
|
void initialize();
|
||||||
|
|
||||||
|
void setStorageProvider(IStorage *pstorage);
|
||||||
|
|
||||||
void trackChanges() { m_fTrackingChanges++; }
|
void trackChanges() { m_fTrackingChanges++; }
|
||||||
void processChanges();
|
void processChanges();
|
||||||
|
|
||||||
@ -1406,6 +1409,16 @@ typedef struct redisDb : public redisDbPersistentDataSnapshot
|
|||||||
using redisDbPersistentData::endSnapshot;
|
using redisDbPersistentData::endSnapshot;
|
||||||
using redisDbPersistentData::consolidate_snapshot;
|
using redisDbPersistentData::consolidate_snapshot;
|
||||||
|
|
||||||
|
dict_iter find_threadsafe(const char *key) const
|
||||||
|
{
|
||||||
|
dict_iter itr = redisDbPersistentDataSnapshot::find_threadsafe(key);
|
||||||
|
if (itr.key() != nullptr && itr.val() == nullptr)
|
||||||
|
{
|
||||||
|
return const_cast<redisDb*>(this)->redisDbPersistentData::find(key);
|
||||||
|
}
|
||||||
|
return itr;
|
||||||
|
}
|
||||||
|
|
||||||
public:
|
public:
|
||||||
expireset::setiter expireitr;
|
expireset::setiter expireitr;
|
||||||
dict *blocking_keys; /* Keys with clients waiting for data (BLPOP)*/
|
dict *blocking_keys; /* Keys with clients waiting for data (BLPOP)*/
|
||||||
|
Loading…
x
Reference in New Issue
Block a user