diff --git a/build.yaml b/build.yaml new file mode 100644 index 000000000..0500f5b7f --- /dev/null +++ b/build.yaml @@ -0,0 +1,48 @@ +# Doc: https://wiki.sc-corp.net/pages/viewpage.action?pageId=121500284 +version: 1 +machamp: + keydb-build: + # Optional - build counter is linked to the build def + tag_template: 0.0.%build.counter% + # Optional - value in seconds before a build is terminated, default is 3600 seconds + timeout: 3600 + # Optional - update ghe or not, default to true + update_ghe: true + code_coverage: false + # Required + steps: + make-build: + type: cmd + # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c + command: ./machamp_scripts/build.sh + tls-test: + type: cmd + parent: make-build + # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c + command: ./runtest --clients $(nproc) --verbose --tls + cluster-test: + type: cmd + parent: make-build + # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c + command: ./runtest-cluster --tls + sentinel-test: + type: cmd + parent: make-build + # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c + command: ./runtest-sentinel + module-test: + type: cmd + parent: make-build + # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c + command: ./runtest-moduleapi + rotation-test: + type: cmd + parent: make-build + # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 + builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c + command: ./runtest-rotation diff --git a/ci.yaml b/ci.yaml new file mode 100644 index 000000000..5614be8e6 --- /dev/null +++ b/ci.yaml @@ -0,0 +1,16 @@ +# Doc: https://wiki.sc-corp.net/display/TOOL/ci.yaml+User+Guide +version: 1 +on: + pull_request: + - workflows: + # All builds that use machamp should use the defined `backend_workflow` + - workflow_type: backend_workflow + # references a build defined in build.yaml + build_name: keydb-build + arch_types: ["amd64", "arm64"] + push: + - branches: [master] + workflows: + - workflow_type: backend_workflow + build_name: keydb-build + arch_types: ["amd64", "arm64"] diff --git a/machamp_scripts/build.sh b/machamp_scripts/build.sh new file mode 100755 index 000000000..e05a9b3bd --- /dev/null +++ b/machamp_scripts/build.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# make the build +git submodule init && git submodule update +make BUILD_TLS=yes -j$(nproc) KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' + +# gen-cert +./utils/gen-test-certs.sh \ No newline at end of file diff --git a/src/StorageCache.cpp b/src/StorageCache.cpp index 2c6dd42b9..d336e2b3d 100644 --- a/src/StorageCache.cpp +++ b/src/StorageCache.cpp @@ -31,11 +31,25 @@ StorageCache::~StorageCache() dictRelease(m_pdict); } -void StorageCache::clear() +void StorageCache::clear(void(callback)(void*)) { std::unique_lock ul(m_lock); if (m_pdict != nullptr) - dictEmpty(m_pdict, nullptr); + dictEmpty(m_pdict, callback); + m_spstorage->clear(); + m_collisionCount = 0; +} + +void StorageCache::clearAsync() +{ + std::unique_lock ul(m_lock); + if (m_pdict != nullptr) { + dict *dSav = m_pdict; + m_pdict = dictCreate(&dbStorageCacheType, nullptr); + g_pserver->asyncworkqueue->AddWorkFunction([dSav]{ + dictEmpty(dSav, nullptr); + }); + } m_spstorage->clear(); m_collisionCount = 0; } diff --git a/src/StorageCache.h b/src/StorageCache.h index 879b512dc..3c38450fb 100644 --- a/src/StorageCache.h +++ b/src/StorageCache.h @@ -38,7 +38,8 @@ public: return cache; } - void clear(); + void clear(void(callback)(void*)); + void clearAsync(); void insert(sds key, const void *data, size_t cbdata, bool fOverwrite); void bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem); void retrieve(sds key, IStorage::callbackSingle fn) const; diff --git a/src/db.cpp b/src/db.cpp index a115c887b..5b67a4198 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -2706,7 +2706,7 @@ void redisDbPersistentData::clear(void(callback)(void*)) delete m_setexpire; m_setexpire = new (MALLOC_LOCAL) expireset(); if (m_spstorage != nullptr) - m_spstorage->clear(); + m_spstorage->clear(callback); dictEmpty(m_pdictTombstone,callback); m_pdbSnapshot = nullptr; } @@ -2926,7 +2926,7 @@ bool redisDbPersistentData::processChanges(bool fSnapshot) if (m_fAllChanged) { if (dictSize(m_pdict) > 0 || m_spstorage->count() > 0) { // in some cases we may have pre-sized the StorageCache's dict, and we don't want clear to ruin it - m_spstorage->clear(); + m_spstorage->clearAsync(); storeDatabase(); } m_fAllChanged = 0; diff --git a/src/lazyfree.cpp b/src/lazyfree.cpp index a05101f2e..fe76b2f4a 100644 --- a/src/lazyfree.cpp +++ b/src/lazyfree.cpp @@ -223,7 +223,7 @@ void redisDbPersistentData::emptyDbAsync() { m_setexpire = new (MALLOC_LOCAL) expireset(); m_pdict = dictCreate(&dbDictType,this); if (m_spstorage != nullptr) - m_spstorage->clear(); + m_spstorage->clearAsync(); if (m_fTrackingChanges) m_fAllChanged = true; atomicIncr(lazyfree_objects,dictSize(oldht1)); diff --git a/src/storage/rocksdb.cpp b/src/storage/rocksdb.cpp index ebe41acc2..7d77cce53 100644 --- a/src/storage/rocksdb.cpp +++ b/src/storage/rocksdb.cpp @@ -132,7 +132,8 @@ size_t RocksDBStorageProvider::clear() auto strName = m_spcolfamily->GetName(); rocksdb::ColumnFamilyHandle *handle = nullptr; - m_spdb->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), strName, &handle); + rocksdb::ColumnFamilyOptions cf_options(m_pfactory->RocksDbOptions()); + m_spdb->CreateColumnFamily(cf_options, strName, &handle); m_spcolfamily = std::shared_ptr(handle); if (!status.ok()) diff --git a/src/storage/rocksdbfactor_internal.h b/src/storage/rocksdbfactor_internal.h index 8a7df2cf5..11862bb7b 100644 --- a/src/storage/rocksdbfactor_internal.h +++ b/src/storage/rocksdbfactor_internal.h @@ -24,6 +24,8 @@ public: virtual size_t filedsRequired() const override; std::string getTempFolder(); + rocksdb::Options RocksDbOptions(); + private: void setVersion(rocksdb::ColumnFamilyHandle*); }; \ No newline at end of file diff --git a/src/storage/rocksdbfactory.cpp b/src/storage/rocksdbfactory.cpp index 8e6ecd0d5..701314d0f 100644 --- a/src/storage/rocksdbfactory.cpp +++ b/src/storage/rocksdbfactory.cpp @@ -39,6 +39,17 @@ IStorageFactory *CreateRocksDBStorageFactory(const char *path, int dbnum, const return new RocksDBStorageFactory(path, dbnum, rgchConfig, cchConfig); } +rocksdb::Options RocksDBStorageFactory::RocksDbOptions() +{ + rocksdb::Options options = DefaultRocksDBOptions(); + options.max_open_files = filedsRequired(); + options.sst_file_manager = m_pfilemanager; + options.create_if_missing = true; + options.create_missing_column_families = true; + options.info_log_level = rocksdb::ERROR_LEVEL; + return options; +} + RocksDBStorageFactory::RocksDBStorageFactory(const char *dbfile, int dbnum, const char *rgchConfig, size_t cchConfig) : m_path(dbfile) { @@ -56,14 +67,10 @@ RocksDBStorageFactory::RocksDBStorageFactory(const char *dbfile, int dbnum, cons m_pfilemanager = std::shared_ptr(rocksdb::NewSstFileManager(rocksdb::Env::Default())); - rocksdb::Options options = DefaultRocksDBOptions(); - options.max_open_files = filedsRequired(); - options.sst_file_manager = m_pfilemanager; - options.create_if_missing = true; - options.create_missing_column_families = true; - options.info_log_level = rocksdb::ERROR_LEVEL; rocksdb::DB *db = nullptr; + auto options = RocksDbOptions(); + for (int idb = 0; idb < dbnum; ++idb) { rocksdb::ColumnFamilyOptions cf_options(options);