Merge branch 'keydbpro' into psync_multimaster_fixes

build.yaml

@@ -0,0 +1,48 @@
+# Doc: https://wiki.sc-corp.net/pages/viewpage.action?pageId=121500284
+version: 1
+machamp:
+  keydb-build:
+    # Optional - build counter is linked to the build def
+    tag_template: 0.0.%build.counter%
+    # Optional - value in seconds before a build is terminated, default is 3600 seconds
+    timeout: 3600
+    # Optional - update ghe or not, default to true
+    update_ghe: true
+    code_coverage: false
+    # Required
+    steps:
+      make-build:
+        type: cmd
+        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
+        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
+        command: ./machamp_scripts/build.sh
+      tls-test:
+        type: cmd
+        parent: make-build
+        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
+        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
+        command: ./runtest --clients $(nproc) --verbose --tls
+      cluster-test:
+        type: cmd
+        parent: make-build
+        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
+        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
+        command: ./runtest-cluster --tls
+      sentinel-test:
+        type: cmd
+        parent: make-build
+        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
+        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
+        command: ./runtest-sentinel
+      module-test:
+        type: cmd
+        parent: make-build
+        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
+        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
+        command: ./runtest-moduleapi
+      rotation-test:
+        type: cmd
+        parent: make-build
+        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
+        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
+        command: ./runtest-rotation

ci.yaml

@@ -0,0 +1,16 @@
+# Doc: https://wiki.sc-corp.net/display/TOOL/ci.yaml+User+Guide
+version: 1
+on:
+  pull_request:
+    - workflows:
+      # All builds that use machamp should use the defined `backend_workflow`
+      - workflow_type: backend_workflow
+        # references a build defined in build.yaml
+        build_name: keydb-build
+        arch_types: ["amd64", "arm64"]
+  push:
+    - branches: [master]
+      workflows:
+        - workflow_type: backend_workflow
+          build_name: keydb-build
+          arch_types: ["amd64", "arm64"]

machamp_scripts/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# make the build
+git submodule init && git submodule update
+make BUILD_TLS=yes -j$(nproc) KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror'
+
+# gen-cert
+./utils/gen-test-certs.sh

src/StorageCache.cpp

@@ -31,11 +31,25 @@ StorageCache::~StorageCache()
         dictRelease(m_pdict);
 }
 
-void StorageCache::clear()
+void StorageCache::clear(void(callback)(void*))
 {
     std::unique_lock<fastlock> ul(m_lock);
     if (m_pdict != nullptr)
-        dictEmpty(m_pdict, nullptr);
+        dictEmpty(m_pdict, callback);
+    m_spstorage->clear();
+    m_collisionCount = 0;
+}
+
+void StorageCache::clearAsync()
+{
+    std::unique_lock<fastlock> ul(m_lock);
+    if (m_pdict != nullptr) {
+        dict *dSav = m_pdict;
+        m_pdict = dictCreate(&dbStorageCacheType, nullptr);
+        g_pserver->asyncworkqueue->AddWorkFunction([dSav]{
+            dictEmpty(dSav, nullptr);
+        });
+    }
     m_spstorage->clear();
     m_collisionCount = 0;
 }

src/StorageCache.h

@@ -38,7 +38,8 @@ public:
         return cache;
     }
 
-    void clear();
+    void clear(void(callback)(void*));
+    void clearAsync();
     void insert(sds key, const void *data, size_t cbdata, bool fOverwrite);
     void bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem);
     void retrieve(sds key, IStorage::callbackSingle fn) const;

src/db.cpp

@@ -2706,7 +2706,7 @@ void redisDbPersistentData::clear(void(callback)(void*))
     delete m_setexpire;
     m_setexpire = new (MALLOC_LOCAL) expireset();
     if (m_spstorage != nullptr)
-        m_spstorage->clear();
+        m_spstorage->clear(callback);
     dictEmpty(m_pdictTombstone,callback);
     m_pdbSnapshot = nullptr;
 }
@@ -2926,7 +2926,7 @@ bool redisDbPersistentData::processChanges(bool fSnapshot)
             if (m_fAllChanged)
             {
                 if (dictSize(m_pdict) > 0 || m_spstorage->count() > 0) { // in some cases we may have pre-sized the StorageCache's dict, and we don't want clear to ruin it
-                    m_spstorage->clear();
+                    m_spstorage->clearAsync();
                     storeDatabase();
                 }
                 m_fAllChanged = 0;

src/lazyfree.cpp

@@ -223,7 +223,7 @@ void redisDbPersistentData::emptyDbAsync() {
     m_setexpire = new (MALLOC_LOCAL) expireset();
     m_pdict = dictCreate(&dbDictType,this);
     if (m_spstorage != nullptr)
-        m_spstorage->clear();
+        m_spstorage->clearAsync();
     if (m_fTrackingChanges)
         m_fAllChanged = true;
     atomicIncr(lazyfree_objects,dictSize(oldht1));

src/storage/rocksdb.cpp

@@ -132,7 +132,8 @@ size_t RocksDBStorageProvider::clear()
     auto strName = m_spcolfamily->GetName();
 
     rocksdb::ColumnFamilyHandle *handle = nullptr;
-    m_spdb->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), strName, &handle);
+    rocksdb::ColumnFamilyOptions cf_options(m_pfactory->RocksDbOptions());
+    m_spdb->CreateColumnFamily(cf_options, strName, &handle);
     m_spcolfamily = std::shared_ptr<rocksdb::ColumnFamilyHandle>(handle);
 
     if (!status.ok())

src/storage/rocksdbfactor_internal.h

@@ -24,6 +24,8 @@ public:
     virtual size_t filedsRequired() const override;
     std::string getTempFolder();
 
+    rocksdb::Options RocksDbOptions();
+
 private:
     void setVersion(rocksdb::ColumnFamilyHandle*);
 };

src/storage/rocksdbfactory.cpp

@@ -39,6 +39,17 @@ IStorageFactory *CreateRocksDBStorageFactory(const char *path, int dbnum, const
     return new RocksDBStorageFactory(path, dbnum, rgchConfig, cchConfig);
 }
 
+rocksdb::Options RocksDBStorageFactory::RocksDbOptions()
+{
+    rocksdb::Options options = DefaultRocksDBOptions();
+    options.max_open_files = filedsRequired();
+    options.sst_file_manager = m_pfilemanager;
+    options.create_if_missing = true;
+    options.create_missing_column_families = true;
+    options.info_log_level = rocksdb::ERROR_LEVEL;
+    return options;
+}
+
 RocksDBStorageFactory::RocksDBStorageFactory(const char *dbfile, int dbnum, const char *rgchConfig, size_t cchConfig)
     : m_path(dbfile)
 {
@@ -56,14 +67,10 @@ RocksDBStorageFactory::RocksDBStorageFactory(const char *dbfile, int dbnum, cons
 
     m_pfilemanager = std::shared_ptr<rocksdb::SstFileManager>(rocksdb::NewSstFileManager(rocksdb::Env::Default()));
 
-    rocksdb::Options options = DefaultRocksDBOptions();
-    options.max_open_files = filedsRequired();
-    options.sst_file_manager = m_pfilemanager;
-    options.create_if_missing = true;
-    options.create_missing_column_families = true;
-    options.info_log_level = rocksdb::ERROR_LEVEL;
     rocksdb::DB *db = nullptr;
 
+    auto options = RocksDbOptions();
+
     for (int idb = 0; idb < dbnum; ++idb)
     {
         rocksdb::ColumnFamilyOptions cf_options(options);