38169682f0
* add docker build * fix the working dir in Dockerfile * add release publish docker image * address intentation and use default release * migrate keydb_modstatsd to keydb-internal * rm * add submodule cpp-statsd-client * include trigger keydb_modstatsd Makefile in modules Makefile * update * have clean also trigger keydb_modstatsd clean * move cpp-statsd-client to deps * checkout to a06a5b9359f31d946fe163b9038586982971ae49 * update relative path in compilation * remove submodule instead use the source * include building keydb statsd module * fix check in Dockerfile docker-entrypoint.sh * fix * fix the comment caused stuck docker build * use commit hash as tag template * fix * test tag * Revert "test tag" This reverts commit 9cbc57137d57aab4fdd5a9283bae07391b3c7f8b. * make docker build independent * add new build to ci * emit system free metrics with '/proc/meminfo' * have emit system free memory within metrics_time_taken_us and also add metric time taken for it * Remove Expireset (#217) Major refactor to place expiry information directly in the object struct. * update MemFree to MemAvailable in keydb statsd * add metric emit for non-empty primary with less than 2 connected replicas * address comments * Multiply CPU percent metric by 100 * Fix memory leaks * Fix slow to free when low lock contention * fix nodename metricsname * fix unnecessary replace * Make propagating before freeing module context optional (#225) * don't propogate on module context free for rdb load * default in wrong place * Flash expiration (#197) Design Doc: https://docs.google.com/document/d/1NmnYGnHLdZp-KOUCUatX5iXpF-L3YK4VUc9Lm3Tqxpo/edit?usp=sharing * Emit more aggregate metrics in modstatsd (#223) * Permit keys of differing slots as long as they are served by this cluster and we are not migrating * Fix over pessimistic checks that prevent replicas from serving mget * Fix logic bug * async rehash is preventing rehashing during RDB load after a db flush. Ensure it can't interefere after a flush * make async rehash configurable * only use internal locks when multithreaded (#205) * Fix crossslot error migrating batches of keys * Fix bug where we erroneously answer queries belonging to another shard * fix mac compile * enable level_compaction_dynamic_level_bytes after flush, and flush expires for FLASH (#229) * enable level_compaction_dynamic_level_bytes after flush, and flush expires * update debug reload for flash * update debug reload for flash complete * missing forward declare * commit existing changes then track changes for debug reload * missing args * commitChanges is conditional Co-authored-by: John Sully <jsully@snapchat.com> --------- Co-authored-by: zliang <zliang@snapchat.com> Co-authored-by: John Sully <jsully@snapchat.com> Co-authored-by: Alex Cope <acope@snapchat.com> Co-authored-by: John Sully <john@csquare.ca>
66 lines
2.9 KiB
C++
66 lines
2.9 KiB
C++
#pragma once
|
|
|
|
#include <memory>
|
|
#include "../IStorage.h"
|
|
#include <rocksdb/db.h>
|
|
#include <rocksdb/utilities/write_batch_with_index.h>
|
|
#include "../fastlock.h"
|
|
|
|
#define INTERNAL_KEY_PREFIX "\x00\x04\x03\x00\x05\x02\x04"
|
|
static const char count_key[] = INTERNAL_KEY_PREFIX "__keydb__count\1";
|
|
static const char version_key[] = INTERNAL_KEY_PREFIX "__keydb__version\1";
|
|
static const char meta_key[] = INTERNAL_KEY_PREFIX "__keydb__metadata\1";
|
|
static const char last_expire_key[] = INTERNAL_KEY_PREFIX "__keydb__last_expire_time";
|
|
class RocksDBStorageFactory;
|
|
|
|
class RocksDBStorageProvider : public IStorage
|
|
{
|
|
RocksDBStorageFactory *m_pfactory;
|
|
std::shared_ptr<rocksdb::DB> m_spdb; // Note: This must be first so it is deleted last
|
|
std::unique_ptr<rocksdb::WriteBatchWithIndex> m_spbatch;
|
|
const rocksdb::Snapshot *m_psnapshot = nullptr;
|
|
std::shared_ptr<rocksdb::ColumnFamilyHandle> m_spcolfamily;
|
|
std::shared_ptr<rocksdb::ColumnFamilyHandle> m_spexpirecolfamily;
|
|
rocksdb::ReadOptions m_readOptionsTemplate;
|
|
size_t m_count = 0;
|
|
mutable fastlock m_lock {"RocksDBStorageProvider"};
|
|
|
|
public:
|
|
RocksDBStorageProvider(RocksDBStorageFactory *pfactory, std::shared_ptr<rocksdb::DB> &spdb, std::shared_ptr<rocksdb::ColumnFamilyHandle> &spcolfam, std::shared_ptr<rocksdb::ColumnFamilyHandle> &spexpirecolfam, const rocksdb::Snapshot *psnapshot, size_t count);
|
|
~RocksDBStorageProvider();
|
|
|
|
virtual void insert(const char *key, size_t cchKey, void *data, size_t cb, bool fOverwrite) override;
|
|
virtual bool erase(const char *key, size_t cchKey) override;
|
|
virtual void retrieve(const char *key, size_t cchKey, callbackSingle fn) const override;
|
|
virtual size_t clear() override;
|
|
virtual bool enumerate(callback fn) const override;
|
|
virtual bool enumerate_hashslot(callback fn, unsigned int hashslot) const override;
|
|
|
|
virtual std::vector<std::string> getExpirationCandidates(unsigned int count) override;
|
|
virtual std::vector<std::string> getEvictionCandidates(unsigned int count) override;
|
|
virtual void setExpire(const char *key, size_t cchKey, long long expire) override;
|
|
virtual void removeExpire(const char *key, size_t cchKey, long long expire) override;
|
|
|
|
virtual const IStorage *clone() const override;
|
|
|
|
virtual void beginWriteBatch() override;
|
|
virtual void endWriteBatch() override;
|
|
|
|
virtual void bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem) override;
|
|
|
|
virtual void batch_lock() override;
|
|
virtual void batch_unlock() override;
|
|
|
|
virtual void flush() override;
|
|
|
|
size_t count() const override;
|
|
|
|
protected:
|
|
bool FKeyExists(std::string&) const;
|
|
bool FExpireExists(std::string&) const;
|
|
|
|
const rocksdb::ReadOptions &ReadOptions() const { return m_readOptionsTemplate; }
|
|
rocksdb::WriteOptions WriteOptions() const;
|
|
};
|
|
|
|
bool FInternalKey(const char *key, size_t cch); |