diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 91b34ebe0..9452c4204 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,20 +1,24 @@ --- name: Bug report -about: Create a report to help us improve -title: '' +about: Help us improve KeyDB by reporting a bug +title: '[BUG]' labels: '' assignees: '' --- **Describe the bug** -A clear and concise description of what the bug is. -** Log Files ** -These should be KeyDB logs, not syslogs or logs from your container manager. If you are reporting a crash there will be a line in your log stating: -"=== KEYDB BUG REPORT START: Cut & paste starting from here ===" +A short description of the bug. -Please copy everything after this line. +**To reproduce** -**To Reproduce** -Do you know how to reproduce this? If so please provide repro steps. +Steps to reproduce the behavior and/or a minimal code sample. + +**Expected behavior** + +A description of what you expected to happen. + +**Additional information** + +Any additional information that is relevant to the problem. diff --git a/.github/ISSUE_TEMPLATE/crash_report.md b/.github/ISSUE_TEMPLATE/crash_report.md new file mode 100644 index 000000000..c608ccdc1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/crash_report.md @@ -0,0 +1,20 @@ +--- +name: Crash report +about: Submit a crash report +title: '[CRASH]' +labels: '' +assignees: '' + +--- + +**Crash report** + +Paste the complete crash log between the quotes below. Please include a few lines from the log preceding the crash report to provide some context. + +``` +``` + +**Aditional information** + +1. OS distribution and version +2. Steps to reproduce (if any) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index bbcbbe7d6..22dfc5631 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,20 +1,24 @@ --- name: Feature request -about: Suggest an idea for this project -title: '' +about: Suggest a feature for KeyDB +title: '[NEW]' labels: '' assignees: '' --- -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] +**The problem/use-case that the feature addresses** -**Describe the solution you'd like** -A clear and concise description of what you want to happen. +A description of the problem that the feature will solve, or the use-case with which the feature will be used. -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. +**Description of the feature** -**Additional context** -Add any other context or screenshots about the feature request here. +A description of what you want to happen. + +**Alternatives you've considered** + +Any alternative solutions or features you've considered, including references to existing open and closed feature requests in this repository. + +**Additional information** + +Any additional information that is relevant to the feature request. diff --git a/.github/ISSUE_TEMPLATE/other_stuff.md b/.github/ISSUE_TEMPLATE/other_stuff.md new file mode 100644 index 000000000..e82cf0024 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/other_stuff.md @@ -0,0 +1,8 @@ +--- +name: Other +about: Can't find the right issue type? Use this one! +title: '' +labels: '' +assignees: '' + +--- diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 000000000..6acde0d06 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,21 @@ +--- +name: Question +about: Ask the Redis developers +title: '[QUESTION]' +labels: '' +assignees: '' + +--- + +Please keep in mind that this issue tracker should be used for reporting bugs or proposing improvements to the Redis server. + +Generally, questions about using Redis should be directed to the [community](https://redis.io/community): + +* [the mailing list](https://groups.google.com/forum/#!forum/redis-db) +* [the `redis` tag at StackOverflow](http://stackoverflow.com/questions/tagged/redis) +* [/r/redis subreddit](http://www.reddit.com/r/redis) +* [the irc channel #redis](http://webchat.freenode.net/?channels=redis) on freenode + +It is also possible that your question was already asked here, so please do a quick issues search before submitting. Lastly, if your question is about one of Redis' [clients](https://redis.io/clients), you may to contact your client's developers for help. + +That said, please feel free to replace all this with your question :) \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb41880c9..4d0f8388f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: run: | sudo apt-get update sudo apt-get -y install uuid-dev libcurl4-openssl-dev - make BUILD_TLS=yes -j2 + make REDIS_CFLAGS='-Werror' BUILD_TLS=yes -j2 - name: gen-cert run: ./utils/gen-test-certs.sh - name: test-tls @@ -44,7 +44,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: make - run: make -j2 + run: make REDIS_CFLAGS='-Werror' -j2 build-libc-malloc: runs-on: ubuntu-latest @@ -54,5 +54,5 @@ jobs: run: | sudo apt-get update sudo apt-get -y install uuid-dev libcurl4-openssl-dev - make MALLOC=libc -j2 + make REDIS_CFLAGS='-Werror' MALLOC=libc -j2 diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml deleted file mode 100644 index 6e4f88ef3..000000000 --- a/.github/workflows/daily.yml +++ /dev/null @@ -1,180 +0,0 @@ -name: Daily - -on: - pull_request: - branches: - # any PR to a release branch. - - '[0-9].[0-9]' - schedule: - - cron: '0 0 * * *' - -jobs: - - test-ubuntu-jemalloc: - runs-on: ubuntu-latest - if: github.repository == 'redis/redis' - timeout-minutes: 14400 - steps: - - uses: actions/checkout@v2 - - name: make - run: | - sudo apt-get -y install uuid-dev libcurl4-openssl-dev - make - - name: test - run: | - sudo apt-get install tcl8.5 - ./runtest --accurate --verbose - - name: module api test - run: ./runtest-moduleapi --verbose - - name: sentinel tests - run: ./runtest-sentinel - - name: cluster tests - run: ./runtest-cluster - - test-ubuntu-libc-malloc: - runs-on: ubuntu-latest - if: github.repository == 'redis/redis' - timeout-minutes: 14400 - steps: - - uses: actions/checkout@v2 - - name: make - run: | - sudo apt-get -y install uuid-dev libcurl4-openssl-dev - make MALLOC=libc - - name: test - run: | - sudo apt-get install tcl8.5 - ./runtest --accurate --verbose - - name: module api test - run: ./runtest-moduleapi --verbose - - name: sentinel tests - run: ./runtest-sentinel - - name: cluster tests - run: ./runtest-cluster - - test: - runs-on: ubuntu-latest - if: github.repository == 'redis/redis' - timeout-minutes: 14400 - steps: - - uses: actions/checkout@v2 - - name: make - run: | - sudo apt-get -y install uuid-dev libcurl4-openssl-dev - make BUILD_TLS=yes -j2 - - name: "test (tls)" - run: | - sudo apt-get install tcl8.5 tcl-tls - ./utils/gen-test-certs.sh - ./runtest --accurate --verbose --tls - - name: test - run: ./runtest --accurate --verbose - - name: module api test (tls) - run: ./runtest-moduleapi --verbose --tls - - test-ubuntu-arm: - runs-on: [self-hosted, linux, arm] - steps: - - uses: actions/checkout@v2 - - name: make - run: | - sudo apt-get -y install uuid-dev libcurl4-openssl-dev - make -j4 - - name: test - run: | - sudo apt-get -y install tcl8.5 - ./runtest --clients 2 --verbose - - name: module tests - run: | - ./runtest-moduleapi - - test-valgrind: - runs-on: ubuntu-latest - if: github.repository == 'redis/redis' - timeout-minutes: 14400 - steps: - - uses: actions/checkout@v2 - - name: make - run: | - sudo apt-get -y install uuid-dev libcurl4-openssl-dev - make valgrind - - name: test - run: | - sudo apt-get update - sudo apt-get install tcl8.5 valgrind -y - ./runtest --valgrind --verbose --clients 1 - - name: module api test - run: ./runtest-moduleapi --valgrind --verbose --clients 1 - - test-centos7-jemalloc: - runs-on: ubuntu-latest - if: github.repository == 'redis/redis' - container: centos:7 - timeout-minutes: 14400 - steps: - - uses: actions/checkout@v2 - - name: make - run: | - yum -y install centos-release-scl - yum -y install devtoolset-7 - scl enable devtoolset-7 "make" - - name: test - run: | - yum -y install tcl - ./runtest --accurate --verbose - - name: module api test - run: ./runtest-moduleapi --verbose - - name: sentinel tests - run: ./runtest-sentinel - - name: cluster tests - run: ./runtest-cluster - - test-centos7-tls: - runs-on: ubuntu-latest - if: github.repository == 'redis/redis' - container: centos:7 - timeout-minutes: 14400 - steps: - - uses: actions/checkout@v2 - - name: make - run: | - yum -y install centos-release-scl epel-release - yum -y install devtoolset-7 openssl-devel openssl - scl enable devtoolset-7 "make BUILD_TLS=yes" - - name: test - run: | - yum -y install tcl tcltls - ./utils/gen-test-certs.sh - ./runtest --accurate --verbose --tls - ./runtest --accurate --verbose - - name: module api test - run: | - ./runtest-moduleapi --verbose --tls - ./runtest-moduleapi --verbose - - name: sentinel tests - run: | - ./runtest-sentinel --tls - ./runtest-sentinel - - name: cluster tests - run: | - ./runtest-cluster --tls - ./runtest-cluster - - test-macos-latest: - runs-on: macos-latest - if: github.repository == 'redis/redis' - timeout-minutes: 14400 - steps: - - uses: actions/checkout@v2 - - name: make - run: make - - name: test - run: | - ./runtest --accurate --verbose --no-latency - - name: module api test - run: ./runtest-moduleapi --verbose - - name: sentinel tests - run: ./runtest-sentinel - - name: cluster tests - run: ./runtest-cluster - diff --git a/.gitignore b/.gitignore index 21f903288..2fa8255df 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,4 @@ Makefile.dep .ccls .ccls-cache/* compile_commands.json +redis.code-workspace diff --git a/00-RELEASENOTES b/00-RELEASENOTES deleted file mode 100644 index bff270e77..000000000 --- a/00-RELEASENOTES +++ /dev/null @@ -1,3280 +0,0 @@ -Redis 6.0 release notes -======================= - --------------------------------------------------------------------------------- -Upgrade urgency levels: - -LOW: No need to upgrade unless there are new features you want to use. -MODERATE: Program an upgrade of the server, but it's not urgent. -HIGH: There is a critical bug that may affect a subset of users. Upgrade! -CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -SECURITY: There are security fixes in the release. --------------------------------------------------------------------------------- - -================================================================================ -Redis 6.0.10 Released Tue Jan 12 16:20:20 IST 2021 -================================================================================ - -Upgrade urgency MODERATE: several bugs with moderate impact are fixed, -Here is a comprehensive list of changes in this release compared to 6.0.9. - -Command behavior changes: -* SWAPDB invalidates WATCHed keys (#8239) -* SORT command behaves differently when used on a writable replica (#8283) -* EXISTS should not alter LRU (#8016) - In Redis 5.0 and 6.0 it would have touched the LRU/LFU of the key. -* OBJECT should not reveal logically expired keys (#8016) - Will now behave the same TYPE or any other non-DEBUG command. -* GEORADIUS[BYMEMBER] can fail with -OOM if Redis is over the memory limit (#8107) - -Other behavior changes: -* Sentinel: Fix missing updates to the config file after SENTINEL SET command (#8229) -* CONFIG REWRITE is atomic and safer, but requires write access to the config file's folder (#7824, #8051) - This change was already present in 6.0.9, but was missing from the release notes. - -Bug fixes with compatibility implications (bugs introduced in Redis 6.0): -* Fix RDB CRC64 checksum on big-endian systems (#8270) - If you're using big-endian please consider the compatibility implications with - RESTORE, replication and persistence. -* Fix wrong order of key/value in Lua's map response (#8266) - If your scripts use redis.setresp() or return a map (new in Redis 6.0), please - consider the implications. - -Bug fixes: -* Fix an issue where a forked process deletes the parent's pidfile (#8231) -* Fix crashes when enabling io-threads-do-reads (#8230) -* Fix a crash in redis-cli after executing cluster backup (#8267) -* Handle output buffer limits for module blocked clients (#8141) - Could result in a module sending reply to a blocked client to go beyond the limit. -* Fix setproctitle related crashes. (#8150, #8088) - Caused various crashes on startup, mainly on Apple M1 chips or under instrumentation. -* Backup/restore cluster mode keys to slots map for repl-diskless-load=swapdb (#8108) - In cluster mode with repl-diskless-load, when loading failed, slot map wouldn't - have been restored. -* Fix oom-score-adj-values range, and bug when used in config file (#8046) - Enabling setting this in the config file in a line after enabling it, would - have been buggy. -* Reset average ttl when empty databases (#8106) - Just causing misleading metric in INFO -* Disable rehash when Redis has child process (#8007) - This could have caused excessive CoW during BGSAVE, replication or AOFRW. -* Further improved ACL algorithm for picking categories (#7966) - Output of ACL GETUSER is now more similar to the one provided by ACL SETUSER. -* Fix bug with module GIL being released prematurely (#8061) - Could in theory (and rarely) cause multi-threaded modules to corrupt memory. -* Reduce effect of client tracking causing feedback loop in key eviction (#8100) -* Fix cluster access to unaligned memory (SIGBUS on old ARM) (#7958) -* Fix saving of strings larger than 2GB into RDB files (#8306) - -Additional improvements: -* Avoid wasteful transient memory allocation in certain cases (#8286, #5954) - -Platform / toolchain support related improvements: -* Fix crash log registers output on ARM. (#8020) -* Add a check for an ARM64 Linux kernel bug (#8224) - Due to the potential severity of this issue, Redis will print log warning on startup. -* Raspberry build fix. (#8095) - -New configuration options: -* oom-score-adj-values config can now take absolute values (besides relative ones) (#8046) - -Module related fixes: -* Moved RMAPI_FUNC_SUPPORTED so that it's usable (#8037) -* Improve timer accuracy (#7987) -* Allow '\0' inside of result of RM_CreateStringPrintf (#6260) - -================================================================================ -Redis 6.0.9 Released Mon Oct 26 10:37:47 IST 2020 -================================================================================ - -Upgrade urgency: SECURITY if you use an affected platform (see below). - Otherwise the upgrade urgency is MODERATE. - -This release fixes a potential heap overflow when using a heap allocator other -than jemalloc or glibc's malloc. See: -https://github.com/redis/redis/pull/7963 - -Other fixes in this release: - -New: -* Memory reporting of clients argv (#7874) -* Add redis-cli control on raw format line delimiter (#7841) -* Add redis-cli support for rediss:// -u prefix (#7900) -* Get rss size support for NetBSD and DragonFlyBSD - -Behavior changes: -* WATCH no longer ignores keys which have expired for MULTI/EXEC (#7920) -* Correct OBJECT ENCODING response for stream type (#7797) -* Allow blocked XREAD on a cluster replica (#7881) -* TLS: Do not require CA config if not used (#7862) - -Bug fixes: -* INFO report real peak memory (before eviction) (#7894) -* Allow requirepass config to clear the password (#7899) -* Fix config rewrite file handling to make it really atomic (#7824) -* Fix excessive categories being displayed from ACLs (#7889) -* Add fsync in replica when full RDB payload was received (#7839) -* Don't write replies to socket when output buffer limit reached (#7202) -* Fix redis-check-rdb support for modules aux data (#7826) -* Other smaller bug fixes - -Modules API: -* Add APIs for version and compatibility checks (#7865) -* Add RM_GetClientCertificate (#7866) -* Add RM_GetDetachedThreadSafeContext (#7886) -* Add RM_GetCommandKeys (#7884) -* Add Swapdb Module Event (#7804) -* RM_GetContextFlags provides indication of being in a fork child (#7783) -* RM_GetContextFlags document missing flags: MULTI_DIRTY, IS_CHILD (#7821) -* Expose real client on connection events (#7867) -* Minor improvements to module blocked on keys (#7903) - -Full list of commits: - -Yossi Gottlieb in commit ce0d74d8f: - Fix wrong zmalloc_size() assumption. (#7963) - 1 file changed, 3 deletions(-) - -Oran Agra in commit d3ef26822: - Attempt to fix sporadic test failures due to wait_for_log_messages (#7955) - 1 file changed, 2 insertions(+) - -David CARLIER in commit 76993a0d4: - cpu affinity: DragonFlyBSD support (#7956) - 2 files changed, 9 insertions(+), 2 deletions(-) - -Zach Fewtrell in commit b23cdc14a: - fix invalid 'failover' identifier in cluster slave selection test (#7942) - 1 file changed, 1 insertion(+), 1 deletion(-) - -WuYunlong in commit 99a4cb401: - Update rdb_last_bgsave_time_sec in INFO on diskless replication (#7917) - 1 file changed, 11 insertions(+), 14 deletions(-) - -Wen Hui in commit 258287c35: - do not add save parameter during config rewrite in sentinel mode (#7945) - 1 file changed, 6 insertions(+) - -Qu Chen in commit 6134279e2: - WATCH no longer ignores keys which have expired for MULTI/EXEC. (#7920) - 2 files changed, 3 insertions(+), 3 deletions(-) - -Oran Agra in commit d15ec67c6: - improve verbose logging on failed test. print log file lines (#7938) - 1 file changed, 4 insertions(+) - -Yossi Gottlieb in commit 8a2e6d24f: - Add a --no-latency tests flag. (#7939) - 5 files changed, 23 insertions(+), 9 deletions(-) - -filipe oliveira in commit 0a1737dc5: - Fixed bug concerning redis-benchmark non clustered benchmark forcing always the same hash tag {tag} (#7931) - 1 file changed, 31 insertions(+), 24 deletions(-) - -Oran Agra in commit 6d9b3df71: - fix 32bit build warnings (#7926) - 2 files changed, 3 insertions(+), 3 deletions(-) - -Wen Hui in commit ed6f7a55e: - fix double fclose in aofrewrite (#7919) - 1 file changed, 6 insertions(+), 5 deletions(-) - -Oran Agra in commit 331d73c92: - INFO report peak memory before eviction (#7894) - 1 file changed, 11 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit e88e13528: - Fix tests failure on busybox systems. (#7916) - 2 files changed, 2 insertions(+), 2 deletions(-) - -Oran Agra in commit b7f53738e: - Allow requirepass config to clear the password (#7899) - 1 file changed, 18 insertions(+), 8 deletions(-) - -Wang Yuan in commit 2ecb28b68: - Remove temporary aof and rdb files in a background thread (#7905) - 2 files changed, 3 insertions(+), 3 deletions(-) - -guybe7 in commit 7bc605e6b: - Minor improvements to module blocked on keys (#7903) - 3 files changed, 15 insertions(+), 9 deletions(-) - -Andreas Lind in commit 1b484608d: - Support redis-cli -u rediss://... (#7900) - 1 file changed, 9 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit 95095d680: - Modules: fix RM_GetCommandKeys API. (#7901) - 3 files changed, 4 insertions(+), 7 deletions(-) - -Meir Shpilraien (Spielrein) in commit cd3ae2f2c: - Add Module API for version and compatibility checks (#7865) - 9 files changed, 180 insertions(+), 3 deletions(-) - -Yossi Gottlieb in commit 1d723f734: - Module API: Add RM_GetClientCertificate(). (#7866) - 6 files changed, 88 insertions(+) - -Yossi Gottlieb in commit d72172752: - Modules: Add RM_GetDetachedThreadSafeContext(). (#7886) - 4 files changed, 52 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit e4f9aff19: - Modules: add RM_GetCommandKeys(). - 6 files changed, 238 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit 6682b913e: - Introduce getKeysResult for getKeysFromCommand. - 7 files changed, 170 insertions(+), 121 deletions(-) - -Madelyn Olson in commit 9db65919c: - Fixed excessive categories being displayed from acls (#7889) - 2 files changed, 29 insertions(+), 2 deletions(-) - -Oran Agra in commit f34c50cf6: - Add some additional signal info to the crash log (#7891) - 1 file changed, 4 insertions(+), 1 deletion(-) - -Oran Agra in commit 300bb4701: - Allow blocked XREAD on a cluster replica (#7881) - 3 files changed, 43 insertions(+) - -Oran Agra in commit bc5cf0f1a: - memory reporting of clients argv (#7874) - 5 files changed, 55 insertions(+), 5 deletions(-) - -DvirDukhan in commit 13d2e6a57: - redis-cli add control on raw format line delimiter (#7841) - 1 file changed, 8 insertions(+), 6 deletions(-) - -Oran Agra in commit d54e25620: - Include internal sds fragmentation in MEMORY reporting (#7864) - 2 files changed, 7 insertions(+), 7 deletions(-) - -Oran Agra in commit ac2c2b74e: - Fix crash in script timeout during AOF loading (#7870) - 2 files changed, 47 insertions(+), 4 deletions(-) - -Rafi Einstein in commit 00d2082e7: - Makefile: enable program suffixes via PROG_SUFFIX (#7868) - 2 files changed, 10 insertions(+), 6 deletions(-) - -nitaicaro in commit d2c2c26e7: - Fixed Tracking test “The other connection is able to get invalidations” (#7871) - 1 file changed, 3 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit 2c172556f: - Modules: expose real client on conn events. - 1 file changed, 11 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit 2972d0c1f: - Module API: Fail ineffective auth calls. - 1 file changed, 5 insertions(+) - -Yossi Gottlieb in commit aeb2a3b6a: - TLS: Do not require CA config if not used. (#7862) - 1 file changed, 5 insertions(+), 3 deletions(-) - -Oran Agra in commit d8e64aeb8: - warning: comparison between signed and unsigned integer in 32bit build (#7838) - 1 file changed, 2 insertions(+), 2 deletions(-) - -David CARLIER in commit 151209982: - Add support for Haiku OS (#7435) - 3 files changed, 16 insertions(+) - -Gavrie Philipson in commit b1d3e169f: - Fix typo in module API docs (#7861) - 1 file changed, 2 insertions(+), 2 deletions(-) - -David CARLIER in commit 08e3b8d13: - getting rss size implementation for netbsd (#7293) - 1 file changed, 20 insertions(+) - -Oran Agra in commit 0377a889b: - Fix new obuf-limits tests to work with TLS (#7848) - 2 files changed, 29 insertions(+), 13 deletions(-) - -caozb in commit a057ad9b1: - ignore slaveof no one in redis.conf (#7842) - 1 file changed, 10 insertions(+), 1 deletion(-) - -Wang Yuan in commit 87ecee645: - Don't support Gopher if enable io threads to read queries (#7851) - 2 files changed, 8 insertions(+), 5 deletions(-) - -Wang Yuan in commit b92902236: - Set 'loading' and 'shutdown_asap' to volatile sig_atomic_t type (#7845) - 1 file changed, 2 insertions(+), 2 deletions(-) - -Uri Shachar in commit ee0875a02: - Fix config rewrite file handling to make it really atomic (#7824) - 1 file changed, 49 insertions(+), 47 deletions(-) - -WuYunlong in commit d577519e1: - Add fsync to readSyncBulkPayload(). (#7839) - 1 file changed, 11 insertions(+) - -Wen Hui in commit 104e0ea3e: - rdb.c: handle fclose error case differently to avoid double fclose (#7307) - 1 file changed, 7 insertions(+), 6 deletions(-) - -Wang Yuan in commit 0eb015ac6: - Don't write replies if close the client ASAP (#7202) - 7 files changed, 144 insertions(+), 2 deletions(-) - -Guy Korland in commit 08a03e32c: - Fix RedisModule_HashGet examples (#6697) - 1 file changed, 4 insertions(+), 4 deletions(-) - -Oran Agra in commit 09551645d: - fix recently broken TLS build error, and add coverage for CI (#7833) - 2 files changed, 4 insertions(+), 3 deletions(-) - -David CARLIER in commit c545ba5d0: - Further NetBSD update and build fixes. (#7831) - 3 files changed, 72 insertions(+), 3 deletions(-) - -WuYunlong in commit ec9050053: - Fix redundancy use of semicolon in do-while macros in ziplist.c. (#7832) - 1 file changed, 3 insertions(+), 3 deletions(-) - -yixiang in commit 27a4d1314: - Fix connGetSocketError usage (#7811) - 2 files changed, 6 insertions(+), 4 deletions(-) - -Oran Agra in commit 30795dcae: - RM_GetContextFlags - document missing flags (#7821) - 1 file changed, 6 insertions(+) - -Yossi Gottlieb in commit 14a12849f: - Fix occasional hangs on replication reconnection. (#7830) - 2 files changed, 14 insertions(+), 3 deletions(-) - -Ariel Shtul in commit d5a1b06dc: - Fix redis-check-rdb support for modules aux data (#7826) - 3 files changed, 21 insertions(+), 1 deletion(-) - -Wen Hui in commit 39f793693: - refactor rewriteStreamObject code for adding missing streamIteratorStop call (#7829) - 1 file changed, 36 insertions(+), 18 deletions(-) - -WuYunlong in commit faad29bfb: - Make IO threads killable so that they can be canceled at any time. - 1 file changed, 1 insertion(+) - -WuYunlong in commit b3f1b5830: - Make main thread killable so that it can be canceled at any time. Refine comment of makeThreadKillable(). - 3 files changed, 11 insertions(+), 4 deletions(-) - -Oran Agra in commit 0f43d1f55: - RM_GetContextFlags provides indication that we're in a fork child (#7783) - 8 files changed, 28 insertions(+), 18 deletions(-) - -Wen Hui in commit a55ea9cdf: - Add Swapdb Module Event (#7804) - 5 files changed, 52 insertions(+) - -Daniel Dai in commit 1d8f72bef: - fix make warnings in debug.c MacOS (#7805) - 2 files changed, 3 insertions(+), 2 deletions(-) - -David CARLIER in commit 556953d93: - debug.c: NetBSD build warning fix. (#7810) - 1 file changed, 4 insertions(+), 3 deletions(-) - -Wang Yuan in commit d02435b66: - Remove tmp rdb file in background thread (#7762) - 6 files changed, 82 insertions(+), 8 deletions(-) - -Oran Agra in commit 1bd7bfdc0: - Add printf attribute and fix warnings and a minor bug (#7803) - 2 files changed, 12 insertions(+), 4 deletions(-) - -WuYunlong in commit d25147b4c: - bio: doFastMemoryTest should try to kill io threads as well. - 3 files changed, 19 insertions(+) - -WuYunlong in commit 4489ba081: - bio: fix doFastMemoryTest. - 4 files changed, 25 insertions(+), 3 deletions(-) - -Wen Hui in commit cf85def67: - correct OBJECT ENCODING response for stream type (#7797) - 1 file changed, 1 insertion(+) - -WuYunlong in commit cf5bcf892: - Clarify help text of tcl scripts. (#7798) - 1 file changed, 1 insertion(+) - -Mykhailo Pylyp in commit f72665c65: - Recalculate hardcoded variables from $::instances_count in sentinel tests (#7561) - 3 files changed, 15 insertions(+), 13 deletions(-) - -Oran Agra in commit c67b19e7a: - Fix failing valgrind installation in github actions (#7792) - 1 file changed, 1 insertion(+) - -Oran Agra in commit 92763fd2a: - fix broken PEXPIREAT test (#7791) - 1 file changed, 10 insertions(+), 6 deletions(-) - -Wang Yuan in commit f5b4c0ccb: - Remove dead global variable 'lru_clock' (#7782) - 1 file changed, 1 deletion(-) - -Oran Agra in commit 82d431fd6: - Squash merging 125 typo/grammar/comment/doc PRs (#7773) - 80 files changed, 436 insertions(+), 416 deletions(-) - -================================================================================ -Redis 6.0.8 Released Wed Sep 09 23:34:17 IDT 2020 -================================================================================ - -Upgrade urgency HIGH: Anyone who's using Redis 6.0.7 with Sentinel or -CONFIG REWRITE command is affected and should upgrade ASAP, see #7760. - -Bug fixes: - -* CONFIG REWRITE after setting oom-score-adj-values either via CONFIG SET or - loading it from a config file, will generate a corrupt config file that will - cause Redis to fail to start -* Fix issue with redis-cli --pipe on MacOS -* Fix RESP3 response for HKEYS/HVALS on non-existing key -* Various small bug fixes - -New features / Changes: - -* Remove THP warning when set to madvise -* Allow EXEC with read commands on readonly replica in cluster -* Add masters/replicas options to redis-cli --cluster call command - -Module API: - -* Add RedisModule_ThreadSafeContextTryLock - -Full list of commits: - -Oran Agra in commit cdabf696a: - Fix RESP3 response for HKEYS/HVALS on non-existing key - 1 file changed, 3 insertions(+), 1 deletion(-) - -Oran Agra in commit ec633c716: - Fix leak in new blockedclient module API test - 1 file changed, 3 insertions(+) - -Yossi Gottlieb in commit 6bac07c5c: - Tests: fix oom-score-adj false positives. (#7772) - 1 file changed, 1 insertion(+), 1 deletion(-) - -杨博东 in commit 6043dc614: - Tests: Add aclfile load and save tests (#7765) - 2 files changed, 41 insertions(+) - -Roi Lipman in commit c0b5f9bf0: - RM_ThreadSafeContextTryLock a non-blocking method for acquiring GIL (#7738) - 7 files changed, 122 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit 5780a1599: - Tests: validate CONFIG REWRITE for all params. (#7764) - 6 files changed, 43 insertions(+), 6 deletions(-) - -Oran Agra in commit e3c14b25d: - Change THP warning to use madvise rather than never (#7771) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Itamar Haber in commit 28929917b: - Documents RM_Call's fmt (#5448) - 1 file changed, 25 insertions(+) - -Jan-Erik Rediger in commit 9146402c2: - Check that THP is not set to always (madvise is ok) (#4001) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Yossi Gottlieb in commit d05089429: - Tests: clean up stale .cli files. (#7768) - 1 file changed, 2 insertions(+) - -Eran Liberty in commit 8861c1bae: - Allow exec with read commands on readonly replica in cluster (#7766) - 3 files changed, 59 insertions(+), 3 deletions(-) - -Yossi Gottlieb in commit 2cf2ff2f6: - Fix CONFIG REWRITE of oom-score-adj-values. (#7761) - 1 file changed, 2 insertions(+), 1 deletion(-) - -Oran Agra in commit 1386c80f7: - handle cur_test for nested tests - 1 file changed, 3 insertions(+) - -Oran Agra in commit c7d4945f0: - Add daily CI for MacOS (#7759) - 1 file changed, 18 insertions(+) - -bodong.ybd in commit 32548264c: - Tests: Some fixes for macOS - 3 files changed, 26 insertions(+), 11 deletions(-) - -Oran Agra in commit 1e17f9812: - Fix cluster consistency-check test (#7754) - 1 file changed, 55 insertions(+), 29 deletions(-) - -Yossi Gottlieb in commit f4ecdf86a: - Tests: fix unmonitored servers. (#7756) - 1 file changed, 5 insertions(+) - -Oran Agra in commit 9f020050d: - fix broken cluster/sentinel tests by recent commit (#7752) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit fdbabb496: - Improve valgrind support for cluster tests (#7725) - 3 files changed, 83 insertions(+), 23 deletions(-) - -Oran Agra in commit 35a6a0bbc: - test infra - add durable mode to work around test suite crashing - 3 files changed, 35 insertions(+), 3 deletions(-) - -Oran Agra in commit e3136b13f: - test infra - wait_done_loading - 2 files changed, 16 insertions(+), 36 deletions(-) - -Oran Agra in commit 83c75dbd9: - test infra - flushall between tests in external mode - 1 file changed, 1 insertion(+) - -Oran Agra in commit 265f5d3cf: - test infra - improve test skipping ability - 3 files changed, 91 insertions(+), 36 deletions(-) - -Oran Agra in commit fcd3a9908: - test infra - reduce disk space usage - 3 files changed, 33 insertions(+), 11 deletions(-) - -Oran Agra in commit b6ea4699f: - test infra - write test name to logfile - 3 files changed, 35 insertions(+) - -Yossi Gottlieb in commit 4a4b07fc6: - redis-cli: fix writeConn() buffer handling. (#7749) - 1 file changed, 37 insertions(+), 6 deletions(-) - -Oran Agra in commit f2d08de2e: - Print server startup messages after daemonization (#7743) - 1 file changed, 4 insertions(+), 4 deletions(-) - -Thandayuthapani in commit 77541d555: - Add masters/replicas options to redis-cli --cluster call command (#6491) - 1 file changed, 13 insertions(+), 2 deletions(-) - -Oran Agra in commit 91d13a854: - fix README about BUILD_WITH_SYSTEMD usage (#7739) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Yossi Gottlieb in commit 88d03d965: - Fix double-make issue with make && make install. (#7734) - 1 file changed, 2 insertions(+) - -================================================================================ -Redis 6.0.7 Released Fri Aug 28 11:05:09 IDT 2020 -================================================================================ - -Upgrade urgency MODERATE: several bugs with moderate impact are fixed, -Specifically the first two listed below which cause protocol errors for clients. - -Bug fixes: - -* CONFIG SET could hung the client when arrives during RDB/ROF loading (When - processed after another command that was also rejected with -LOADING error) -* LPOS command when RANK is greater than matches responded wiht broken protocol - (negative multi-bulk count) -* UNLINK / Lazyfree for stream type key would have never do async freeing -* PERSIST should invalidate WATCH (Like EXPIRE does) -* EXEC with only read commands could have be rejected when OOM -* TLS: relax verification on CONFIG SET (Don't error if some configs are set - and tls isn't enabled) -* TLS: support cluster/replication without tls-port -* Systemd startup after network is online -* Redis-benchmark improvements -* Various small bug fixes - -New features: - -* Add oom-score-adj configuration option to control Linux OOM killer -* Show IO threads statistics and status in INFO output -* Add optional tls verification mode (see tls-auth-clients) - -Module API: - -* Add RedisModule_HoldString -* Add loaded keyspace event -* Fix RedisModuleEvent_LoadingProgress -* Fix RedisModuleEvent_MasterLinkChange hook missing on successful psync -* Fix missing RM_CLIENTINFO_FLAG_SSL -* Refactor redismodule.h for use with -fno-common / extern - -Full list of commits: - -Oran Agra in commit c26394e4f: - Reduce the probability of failure when start redis in runtest-cluster #7554 (#7635) - 1 file changed, 23 insertions(+), 5 deletions(-) - -Leoš Literák in commit 745d5e802: - Update README.md with instructions how to build with systemd support (#7730) - 1 file changed, 5 insertions(+) - -Yossi Gottlieb in commit 03f1d208a: - Fix oom-score-adj on older distros. (#7724) - 1 file changed, 2 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit 941174d9c: - Backport Lua 5.2.2 stack overflow fix. (#7733) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Wang Yuan in commit c897dba14: - Fix wrong format specifiers of 'sdscatfmt' for the INFO command (#7706) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Wen Hui in commit 5e3fab5e7: - fix make warnings (#7692) - 1 file changed, 4 insertions(+), 3 deletions(-) - -Nathan Scott in commit a2b09c13f: - Annotate module API functions in redismodule.h for use with -fno-common (#6900) - 1 file changed, 265 insertions(+), 241 deletions(-) - -Yossi Gottlieb in commit bf244273f: - Add oom-score-adj configuration option to control Linux OOM killer. (#1690) - 8 files changed, 306 insertions(+), 1 deletion(-) - -Meir Shpilraien (Spielrein) in commit b5a6ab98f: - see #7544, added RedisModule_HoldString api. (#7577) - 4 files changed, 83 insertions(+), 8 deletions(-) - -ShooterIT in commit ff04cf62b: - [Redis-benchmark] Remove zrem test, add zpopmin test - 1 file changed, 5 insertions(+), 5 deletions(-) - -ShooterIT in commit 0f3260f31: - [Redis-benchmark] Support zset type - 1 file changed, 16 insertions(+) - -Arun Ranganathan in commit 45d0b94fc: - Show threading configuration in INFO output (#7446) - 3 files changed, 46 insertions(+), 14 deletions(-) - -Meir Shpilraien (Spielrein) in commit a22f61e12: - This PR introduces a new loaded keyspace event (#7536) - 8 files changed, 135 insertions(+), 4 deletions(-) - -Oran Agra in commit 1c9ca1030: - Fix rejectCommand trims newline in shared error objects, hung clients (#7714) - 4 files changed, 42 insertions(+), 23 deletions(-) - -valentinogeron in commit 217471795: - EXEC with only read commands should not be rejected when OOM (#7696) - 2 files changed, 51 insertions(+), 8 deletions(-) - -Itamar Haber in commit 6e6c47d16: - Expands lazyfree's effort estimate to include Streams (#5794) - 1 file changed, 24 insertions(+) - -Yossi Gottlieb in commit da6813623: - Add language servers stuff, test/tls to gitignore. (#7698) - 1 file changed, 4 insertions(+) - -Valentino Geron in commit de7fb126e: - Assert that setDeferredAggregateLen isn't called with negative value - 1 file changed, 1 insertion(+) - -Valentino Geron in commit 6cf27f25f: - Fix LPOS command when RANK is greater than matches - 2 files changed, 9 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit 9bba54ace: - Tests: fix redis-cli with remote hosts. (#7693) - 3 files changed, 5 insertions(+), 5 deletions(-) - -huangzhw in commit 0fec2cb81: - RedisModuleEvent_LoadingProgress always at 100% progress (#7685) - 1 file changed, 2 insertions(+), 2 deletions(-) - -guybe7 in commit 931e19aa6: - Modules: Invalidate saved_oparray after use (#7688) - 1 file changed, 2 insertions(+) - -杨博东 in commit 6f2065570: - Fix flock cluster config may cause failure to restart after kill -9 (#7674) - 4 files changed, 31 insertions(+), 7 deletions(-) - -Raghav Muddur in commit 200149a2a: - Update clusterMsgDataPublish to clusterMsgModule (#7682) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Madelyn Olson in commit 72daa1b4e: - Fixed hset error since it's shared with hmset (#7678) - 1 file changed, 1 insertion(+), 1 deletion(-) - -guybe7 in commit 3bf9ac994: - PERSIST should signalModifiedKey (Like EXPIRE does) (#7671) - 1 file changed, 1 insertion(+) - -Oran Agra in commit b37501684: - OOM Crash log include size of allocation attempt. (#7670) - 1 file changed, 2 insertions(+), 1 deletion(-) - -Wen Hui in commit 2136cb68f: - [module] using predefined REDISMODULE_NO_EXPIRE in RM_GetExpire (#7669) - 1 file changed, 2 insertions(+), 1 deletion(-) - -Oran Agra in commit f56aee4bc: - Trim trailing spaces in error replies coming from rejectCommand (#7668) - 1 file changed, 5 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit 012d7506a: - Module API: fix missing RM_CLIENTINFO_FLAG_SSL. (#7666) - 6 files changed, 82 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit a0adbc857: - TLS: relax verification on CONFIG SET. (#7665) - 2 files changed, 24 insertions(+), 7 deletions(-) - -Madelyn Olson in commit 2ef29715b: - Fixed timer warning (#5953) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Wagner Francisco Mezaroba in commit b76f171f5: - allow --pattern to be used along with --bigkeys (#3586) - 1 file changed, 9 insertions(+), 2 deletions(-) - -zhaozhao.zz in commit cc7b57765: - redis-benchmark: fix wrong random key for hset (#4895) - 1 file changed, 1 insertion(+), 1 deletion(-) - -zhaozhao.zz in commit 479c1ba77: - CLIENT_MASTER should ignore server.proto_max_bulk_len - 1 file changed, 2 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit f61ce8a52: - config: proto-max-bulk-len must be 1mb or greater - 2 files changed, 2 insertions(+), 2 deletions(-) - -zhaozhao.zz in commit 0350f597a: - using proto-max-bulk-len in checkStringLength for SETRANGE and APPEND - 1 file changed, 2 insertions(+), 2 deletions(-) - -YoongHM in commit eea63548d: - Start redis after network is online (#7639) - 1 file changed, 2 insertions(+) - -Yossi Gottlieb in commit aef6d74fb: - Run daily workflow on main repo only (no forks). (#7646) - 1 file changed, 7 insertions(+) - -WuYunlong in commit 917b4d241: - see #7250, fix signature of RedisModule_DeauthenticateAndCloseClient (#7645) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Wang Yuan in commit efab7fd54: - Print error info if failed opening config file (#6943) - 1 file changed, 2 insertions(+), 1 deletion(-) - -Wen Hui in commit 8c4468bcf: - fix memory leak in ACLLoadFromFile error handling (#7623) - 1 file changed, 1 insertion(+) - -Oran Agra in commit 89724e1d2: - redis-cli --cluster-yes - negate force flag for clarity - 1 file changed, 9 insertions(+), 9 deletions(-) - -Frank Meier in commit c813739af: - reintroduce REDISCLI_CLUSTER_YES env variable in redis-cli - 1 file changed, 6 insertions(+) - -Frank Meier in commit 7e3b86c18: - add force option to 'create-cluster create' script call (#7612) - 1 file changed, 6 insertions(+), 2 deletions(-) - -Oran Agra in commit 3f7fa4312: - fix new rdb test failing on timing issues (#7604) - 1 file changed, 2 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit 417976d7a: - Fix test-centos7-tls daily job. (#7598) - 1 file changed, 2 insertions(+), 2 deletions(-) - -Oran Agra in commit c41818c51: - module hook for master link up missing on successful psync (#7584) - 2 files changed, 22 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit 6ef3fc185: - CI: Add daily CentOS 7.x jobs. (#7582) - 1 file changed, 50 insertions(+), 4 deletions(-) - -WuYunlong in commit 002c37482: - Fix running single test 14-consistency-check.tcl (#7587) - 1 file changed, 1 insertion(+) - -Yossi Gottlieb in commit 66cbbb6ad: - Clarify RM_BlockClient() error condition. (#6093) - 1 file changed, 9 insertions(+) - -namtsui in commit 22aba2207: - Avoid an out-of-bounds read in the redis-sentinel (#7443) - 1 file changed, 2 insertions(+), 2 deletions(-) - -Wen Hui in commit af08887dc: - Add SignalModifiedKey hook in XGROUP CREATE with MKSTREAM option (#7562) - 1 file changed, 1 insertion(+) - -Wen Hui in commit a5e0a64b0: - fix leak in error handling of debug populate command (#7062) - 1 file changed, 3 insertions(+), 4 deletions(-) - -Yossi Gottlieb in commit cbfdfa231: - Fix TLS cluster tests. (#7578) - 1 file changed, 4 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit 6d5376d30: - TLS: Propagate and handle SSL_new() failures. (#7576) - 4 files changed, 48 insertions(+), 6 deletions(-) - -Oran Agra in commit a662cd577: - Fix failing tests due to issues with wait_for_log_message (#7572) - 3 files changed, 38 insertions(+), 34 deletions(-) - -Jiayuan Chen in commit 2786a4b5e: - Add optional tls verification (#7502) - 6 files changed, 40 insertions(+), 5 deletions(-) - -Oran Agra in commit 3ef3d3612: - Daily github action: run cluster and sentinel tests with tls (#7575) - 1 file changed, 2 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit f20f63322: - TLS: support cluster/replication without tls-port. - 2 files changed, 5 insertions(+), 4 deletions(-) - -grishaf in commit 3c9ae059d: - Fix prepareForShutdown function declaration (#7566) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit 3f4803af9: - Stabilize bgsave test that sometimes fails with valgrind (#7559) - 1 file changed, 20 insertions(+), 2 deletions(-) - -Madelyn Olson in commit 1a3c51a1f: - Properly reset errno for rdbLoad (#7542) - 1 file changed, 1 insertion(+) - -Oran Agra in commit 92d80b13a: - testsuite may leave servers alive on error (#7549) - 1 file changed, 3 insertions(+) - -Yossi Gottlieb in commit 245582ba7: - Tests: drop TCL 8.6 dependency. (#7548) - 1 file changed, 27 insertions(+), 22 deletions(-) - -Oran Agra in commit f20e1ba2d: - Fixes to release scripts (#7547) - 2 files changed, 2 insertions(+), 2 deletions(-) - -Remi Collet in commit 60ff56993: - Fix deprecated tail syntax in tests (#7543) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Wen Hui in commit 34e8541b9: - Add missing calls to raxStop (#7532) - 4 files changed, 63 insertions(+), 19 deletions(-) - -Wen Hui in commit 2f7bc5435: - add missing caching command in client help (#7399) - 1 file changed, 1 insertion(+) - -zhaozhao.zz in commit c15be9ffe: - replication: need handle -NOPERM error after send ping (#7538) - 1 file changed, 1 insertion(+) - -Scott Brenner in commit 1b29152c3: - GitHub Actions workflows - use latest version of actions/checkout (#7534) - 2 files changed, 10 insertions(+), 10 deletions(-) - -================================================================================ -Redis 6.0.6 Released Mon Jul 20 09:31:30 IDT 2020 -================================================================================ - -Upgrade urgency MODERATE: several bugs with moderate impact are fixed here. - -The most important issues are listed here: - -* Fix crash when enabling CLIENT TRACKING with prefix -* EXEC always fails with EXECABORT and multi-state is cleared -* RESTORE ABSTTL won't store expired keys into the db -* redis-cli better handling of non-pritable key names -* TLS: Ignore client cert when tls-auth-clients off -* Tracking: fix invalidation message on flush -* Notify systemd on Sentinel startup -* Fix crash on a misuse of STRALGO -* Few fixes in module API -* Fix a few rare leaks (STRALGO error misuse, Sentinel) -* Fix a possible invalid access in defrag of scripts (unlikely to cause real harm) - -New features: - -* LPOS command to search in a list -* Use user+pass for MIGRATE in redis-cli and redis-benchmark in cluster mode -* redis-cli support TLS for --pipe, --rdb and --replica options -* TLS: Session caching configuration support - -And this is the full list of commits: - -Itamar Haber in commit 50548cafc: - Adds SHA256SUM to redis-stable tarball upload - 1 file changed, 1 insertion(+) - -yoav-steinberg in commit 3a4c6684f: - Support passing stack allocated module strings to moduleCreateArgvFromUserFormat (#7528) - 1 file changed, 4 insertions(+), 1 deletion(-) - -Luke Palmer in commit 2fd0b2bd6: - Send null for invalidate on flush (#7469) - 1 file changed, 14 insertions(+), 10 deletions(-) - -dmurnane in commit c3c81e1a8: - Notify systemd on sentinel startup (#7168) - 1 file changed, 4 insertions(+) - -Developer-Ecosystem-Engineering in commit e2770f29b: - Add registers dump support for Apple silicon (#7453) - 1 file changed, 54 insertions(+), 2 deletions(-) - -Wen Hui in commit b068eae97: - correct error msg for num connections reaching maxclients in cluster mode (#7444) - 1 file changed, 2 insertions(+), 2 deletions(-) - -WuYunlong in commit e6169ae5c: - Fix command help for unexpected options (#7476) - 6 files changed, 20 insertions(+), 3 deletions(-) - -WuYunlong in commit abf08fc02: - Refactor RM_KeyType() by using macro. (#7486) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit 11b83076a: - diskless master disconnect replicas when rdb child failed (#7518) - 1 file changed, 6 insertions(+), 5 deletions(-) - -Oran Agra in commit 8f27f2f7d: - redis-cli tests, fix valgrind timing issue (#7519) - 1 file changed, 1 insertion(+), 1 deletion(-) - -WuYunlong in commit 180b588e8: - Fix out of update help info in tcl tests. (#7516) - 1 file changed, 2 deletions(-) - -Qu Chen in commit 417c60bdc: - Replica always reports master's config epoch in CLUSTER NODES output. (#7235) - 1 file changed, 5 insertions(+), 1 deletion(-) - -Oran Agra in commit 72a242419: - RESTORE ABSTTL skip expired keys - leak (#7511) - 1 file changed, 1 insertion(+) - -Oran Agra in commit 2ca45239f: - fix recently added time sensitive tests failing with valgrind (#7512) - 2 files changed, 12 insertions(+), 6 deletions(-) - -Oran Agra in commit 123dc8b21: - runtest --stop pause stops before terminating the redis server (#7513) - 2 files changed, 8 insertions(+), 2 deletions(-) - -Oran Agra in commit a6added45: - update release scripts for new hosts, and CI to run more tests (#7480) - 5 files changed, 68 insertions(+), 26 deletions(-) - -jimgreen2013 in commit cf4869f9e: - fix description about ziplist, the code is ok (#6318) - 1 file changed, 2 insertions(+), 2 deletions(-) - -马永泽 in commit d548f219b: - fix benchmark in cluster mode fails to authenticate (#7488) - 1 file changed, 56 insertions(+), 40 deletions(-) - -Abhishek Soni in commit e58eb7b89: - fix: typo in CI job name (#7466) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Jiayuan Chen in commit 6def10a2b: - Fix typo in deps README (#7500) - 1 file changed, 1 insertion(+), 1 deletion(-) - -WuYunlong in commit 8af61afef: - Add missing latency-monitor tcl test to test_helper.tcl. (#6782) - 1 file changed, 1 insertion(+) - -Yossi Gottlieb in commit a419f400e: - TLS: Session caching configuration support. (#7420) - 6 files changed, 56 insertions(+), 16 deletions(-) - -Yossi Gottlieb in commit 2e4bb2667: - TLS: Ignore client cert when tls-auth-clients off. (#7457) - 1 file changed, 1 insertion(+), 3 deletions(-) - -James Hilliard in commit f0b1aee9e: - Use pkg-config to properly detect libssl and libcrypto libraries (#7452) - 1 file changed, 15 insertions(+), 3 deletions(-) - -Yossi Gottlieb in commit e92b99564: - TLS: Add missing redis-cli options. (#7456) - 3 files changed, 166 insertions(+), 52 deletions(-) - -Oran Agra in commit 1f3db5bf5: - redis-cli --hotkeys fixed to handle non-printable key names - 1 file changed, 11 insertions(+), 5 deletions(-) - -Oran Agra in commit c3044f369: - redis-cli --bigkeys fixed to handle non-printable key names - 1 file changed, 24 insertions(+), 16 deletions(-) - -Oran Agra in commit b3f75527b: - RESTORE ABSTTL won't store expired keys into the db (#7472) - 4 files changed, 46 insertions(+), 16 deletions(-) - -huangzhw in commit 6f87fc92f: - defrag.c activeDefragSdsListAndDict when defrag sdsele, We can't use (#7492) - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit d8e6a3e5b: - skip a test that uses +inf on valgrind (#7440) - 1 file changed, 12 insertions(+), 9 deletions(-) - -Oran Agra in commit 28fd1a110: - stabilize tests that look for log lines (#7367) - 3 files changed, 33 insertions(+), 11 deletions(-) - -Oran Agra in commit a513b4ed9: - tests/valgrind: don't use debug restart (#7404) - 4 files changed, 114 insertions(+), 57 deletions(-) - -Oran Agra in commit 70e72fc1b: - change references to the github repo location (#7479) - 5 files changed, 7 insertions(+), 7 deletions(-) - -zhaozhao.zz in commit c63e533cc: - BITOP: propagate only when it really SET or DEL targetkey (#5783) - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 31040ff54: - Update comment to clarify change in #7398. - 1 file changed, 4 insertions(+), 1 deletion(-) - -antirez in commit b605fe827: - LPOS: option FIRST renamed RANK. - 2 files changed, 19 insertions(+), 19 deletions(-) - -Dave Nielsen in commit 8deb24954: - updated copyright year - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit a61c2930c: - EXEC always fails with EXECABORT and multi-state is cleared - 6 files changed, 204 insertions(+), 91 deletions(-) - -antirez in commit 3c8041637: - Include cluster.h for getClusterConnectionsCount(). - 1 file changed, 1 insertion(+) - -antirez in commit 5be673ee8: - Fix BITFIELD i64 type handling, see #7417. - 1 file changed, 8 insertions(+), 6 deletions(-) - -antirez in commit 5f289df9b: - Clarify maxclients and cluster in conf. Remove myself too. - 2 files changed, 9 insertions(+), 1 deletion(-) - -hwware in commit 000f928d6: - fix memory leak in sentinel connection sharing - 1 file changed, 1 insertion(+) - -chenhui0212 in commit d9a3c0171: - Fix comments in function raxLowWalk of listpack.c - 1 file changed, 2 insertions(+), 2 deletions(-) - -Tomasz Poradowski in commit 7526e4506: - ensure SHUTDOWN_NOSAVE in Sentinel mode - 2 files changed, 9 insertions(+), 8 deletions(-) - -chenhui0212 in commit 6487cbc33: - fix comments in listpack.c - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit 69b66bfca: - Use cluster connections too, to limit maxclients. - 3 files changed, 23 insertions(+), 8 deletions(-) - -antirez in commit 5a960a033: - Tracking: fix enableBcastTrackingForPrefix() invalid sdslen() call. - 1 file changed, 1 insertion(+), 1 deletion(-) - -root in commit 1c2e50de3: - cluster.c remove if of clusterSendFail in markNodeAsFailingIfNeeded - 1 file changed, 1 insertion(+), 1 deletion(-) - -meir@redislabs.com in commit 040efb697: - Fix RM_ScanKey module api not to return int encoded strings - 3 files changed, 24 insertions(+), 7 deletions(-) - -antirez in commit 1b8b7941d: - Fix LCS object type checking. Related to #7379. - 1 file changed, 17 insertions(+), 10 deletions(-) - -hwware in commit 6b571b45a: - fix memory leak - 1 file changed, 11 insertions(+), 12 deletions(-) - -hwware in commit 674759062: - fix server crash in STRALGO command - 1 file changed, 7 insertions(+) - -Benjamin Sergeant in commit a05ffefdc: - Update redis-cli.c - 1 file changed, 19 insertions(+), 6 deletions(-) - -Jamie Scott in commit 870b63733: - minor fix - 1 file changed, 2 insertions(+), 3 deletions(-) - -================================================================================ -Redis 6.0.5 Released Tue Jun 09 11:56:08 CEST 2020 -================================================================================ - -Upgrade urgency MODERATE: several bugs with moderate impact are fixed here. - -The most important issues are listed here: - -* Fix handling of speical chars in ACL LOAD. -* Make Redis Cluster more robust about operation errors that may lead - to two clusters to mix together. -* Revert the sendfile() implementation of RDB transfer. It causes some delay. -* Fix TLS certificate loading for chained certificates. -* Fix AOF rewirting of KEEPTTL SET option. -* Fix MULTI/EXEC behavior during -BUSY script errors. - -And this is the full list of commits: - -antirez in commit ee8dd01bb: - Temporary fix for #7353 issue about EVAL during -BUSY. - 1 file changed, 9 insertions(+) - -xhe in commit a4a856d53: - return the correct proto version HELLO should return the current proto version, while the code hardcoded 3 - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit e2046b300: - Don't queue commands in an already aborted MULTI state - 1 file changed, 7 insertions(+) - -Oran Agra in commit b35fdf1de: - Avoid rejecting WATCH / UNWATCH, like MULTI/EXEC/DISCARD - 1 file changed, 4 insertions(+), 2 deletions(-) - -zhaozhao.zz in commit 1d7bf208c: - AOF: append origin SET if no expire option - 2 files changed, 23 insertions(+), 8 deletions(-) - -Oran Agra in commit 676445ad9: - fix disconnectSlaves, to try to free each slave. - 1 file changed, 1 deletion(-) - -zhaozhao.zz in commit 4846c0c8a: - donot free protected client in freeClientsInAsyncFreeQueue - 1 file changed, 9 insertions(+), 3 deletions(-) - -Oran Agra in commit f33de403e: - fix pingoff test race - 1 file changed, 1 insertion(+) - -Kevin Fwu in commit 49af4d07e: - Fix TLS certificate loading for chained certificates. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 329fddbda: - Revert "Implements sendfile for redis." - 2 files changed, 2 insertions(+), 55 deletions(-) - -antirez in commit 925a2cd5a: - Revert "avoid using sendfile if tls-replication is enabled" - 1 file changed, 27 insertions(+), 34 deletions(-) - -Liu Zhen in commit 84a7a9058: - fix clusters mixing accidentally by gossip - 1 file changed, 10 insertions(+), 2 deletions(-) - -antirez in commit cd63359a1: - Fix handling of special chars in ACL LOAD. - 1 file changed, 8 insertions(+), 4 deletions(-) - -================================================================================ -Redis 6.0.4 Released Thu May 28 11:36:45 CEST 2020 -================================================================================ - -Upgrade urgency CRITICAL: this release fixes a severe replication bug. - -Redis 6.0.4 fixes a critical replication bug caused by a new feature introduced -in Redis 6. The feature, called "meaningful offset" and strongly wanted by -myself (antirez) was an improvement that avoided that masters were no longer -able, during a failover where they were demoted to replicas, to partially -synchronize with the new master. In short the feature was able to avoid full -synchronizations with RDB. How did it work? By trimming the replication backlog -of the final "PING" commands the master was sending in the replication channel: -this way the replication offset would no longer go "after" the one of the -promoted replica, allowing the master to just continue in the same replication -history, receiving only a small data difference. - -However after the introduction of the feature we (the Redis core team) quickly -understood there was something wrong: the apparently harmless feature had -many bugs, and the last bug we discovered, after a joined effort of multiple -people, we were not even able to fully understand after fixing it. Enough was -enough, we decided that the complexity cost of this feature was too high. -So Redis 6.0.4 removes the feature entirely, and fixes the data corruption that -it was able to cause. - -However there are two facts to take in mind. - -Fact 1: Setups using chained replication, that means that certain replicas -are replicating from other replicas, up to Redis 6.0.3 can experience data -corruption. For chained replication we mean that: - - +--------+ +---------+ +-------------+ - | master |--------->| replica |-------->| sub-replica | - +--------+ +---------+ +-------------+ - - -People using chained replication SHOULD UPGRADE ASAP away from Redis 6.0.0, -6.0.1, 6.0.2 or 6.0.3 to Redis 6.0.4. - -To be clear, people NOT using this setup, but having just replicas attached -directly to the master, SHOUDL NOT BE in danger of any problem. But we -are no longer confident on 6.0.x replication implementation complexities -so we suggest to upgrade to 6.0.4 to everybody using an older 6.0.3 release. -We just so far didn't find any bug that affects Redis 6.0.3 that does not -involve chained replication. - -People starting with Redis 6.0.4 are fine. People with Redis 5 are fine. -People upgrading from Redis 5 to Redis 6.0.4 are fine. -TLDR: The problem is with users of 6.0.0, 6.0.1, 6.0.2, 6.0.3. - -Fact 2: Upgrading from Redis 6.0.x to Redis 6.0.4, IF AND ONLY IF you -use chained replication, requires some extra care: - -1. Once you attach your new Redis 6.0.4 instance as a replica of the current - Redis 6.0.x master, you should wait for the first full synchronization, - then you should promote it right away, if your setup involves chained - replication. Don't give it the time to do a new partial synchronization - in the case the link between the master and the replica will break in - the mean time. - -2. As an additional care, you may want to set the replication ping period - to a very large value (for instance 1000000) using the following command: - - CONFIG SET repl-ping-replica-period 1000000 - - Note that if you do "1" with care, "2" is not needed. - However if you do it, make sure to later restore it to its default: - - CONFIG SET repl-ping-replica-period 10 - -So this is the main change in Redis 6. Later we'll find a different way in -order to achieve what we wanted to achieve with the Meaningful Offset feature, -but without the same complexity. - -Other changes in this release: - -* PSYNC2 tests improved. -* Fix a rare active defrag edge case bug leading to stagnation -* Fix Redis 6 asserting at startup in 32 bit systems. -* Redis 6 32 bit is now added back to our testing environments. -* Fix server crash for STRALGO command, -* Implement sendfile for RDB transfer. -* TLS fixes. -* Make replication more resistant by disconnecting the master if we - detect a protocol error. Basically we no longer accept inline protocol - from the master. -* Other improvements in the tests. - -Regards, -antirez - -This is the full list of commits: - -antirez in commit 59cd4c9f6: - Test: take PSYNC2 test master timeout high during switch. - 1 file changed, 1 deletion(-) - -antirez in commit 6c1bb7b19: - Test: add the tracking unit as default. - 1 file changed, 1 insertion(+) - -Oran Agra in commit 1aee695e5: - tests: find_available_port start search from next port - 1 file changed, 12 insertions(+), 7 deletions(-) - -Oran Agra in commit a2ae46352: - tests: each test client work on a distinct port range - 5 files changed, 39 insertions(+), 27 deletions(-) - -Oran Agra in commit 86e562d69: - 32bit CI needs to build modules correctly - 2 files changed, 7 insertions(+), 2 deletions(-) - -Oran Agra in commit ab2984b1e: - adjust revived meaningful offset tests - 1 file changed, 39 insertions(+), 20 deletions(-) - -Oran Agra in commit 1ff5a222d: - revive meaningful offset tests - 2 files changed, 213 insertions(+) - -antirez in commit cc549b46a: - Replication: showLatestBacklog() refactored out. - 3 files changed, 36 insertions(+), 25 deletions(-) - -antirez in commit 377dd0515: - Drop useless line from replicationCacheMaster(). - 1 file changed, 2 deletions(-) - -antirez in commit 3f8d113f1: - Another meaningful offset test removed. - 1 file changed, 100 deletions(-) - -antirez in commit d4541349d: - Remove the PSYNC2 meaningful offset test. - 2 files changed, 113 deletions(-) - -antirez in commit 2112a5702: - Remove the meaningful offset feature. - 4 files changed, 10 insertions(+), 93 deletions(-) - -antirez in commit d2eb6e0b4: - Set a protocol error if master use the inline protocol. - 1 file changed, 17 insertions(+), 2 deletions(-) - -Oran Agra in commit 9c1df3b76: - daily CI test with tls - 1 file changed, 15 insertions(+) - -Oran Agra in commit 115ed1911: - avoid using sendfile if tls-replication is enabled - 1 file changed, 34 insertions(+), 27 deletions(-) - -antirez in commit 11c748aac: - Replication: log backlog creation event. - 1 file changed, 3 insertions(+) - -antirez in commit 8f1013722: - Test: PSYNC2 test can now show server logs. - 1 file changed, 88 insertions(+), 25 deletions(-) - -antirez in commit 2e591fc4a: - Clarify what is happening in PR #7320. - 1 file changed, 5 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit cbb51fb8f: - PSYNC2: second_replid_offset should be real meaningful offset - 1 file changed, 3 insertions(+), 3 deletions(-) - -Oran Agra in commit e0fc88b4d: - add CI for 32bit build - 2 files changed, 34 insertions(+) - -antirez in commit e3f864b5f: - Make disconnectSlaves() synchronous in the base case. - 3 files changed, 20 insertions(+), 9 deletions(-) - -ShooterIT in commit 8af1e513f: - Implements sendfile for redis. - 2 files changed, 55 insertions(+), 2 deletions(-) - -antirez in commit 3c21418cd: - Fix #7306 less aggressively. - 2 files changed, 29 insertions(+), 17 deletions(-) - -Madelyn Olson in commit e201f83ce: - EAGAIN for tls during diskless load - 1 file changed, 4 insertions(+) - -Qu Chen in commit 58fc456cb: - Disconnect chained replicas when the replica performs PSYNC with the master always to avoid replication offset mismatch between master and chained replicas. - 2 files changed, 60 insertions(+), 3 deletions(-) - -hwware in commit 3febc5c29: - using moreargs variable - 1 file changed, 2 insertions(+), 2 deletions(-) - -hwware in commit 8d6738559: - fix server crash for STRALGO command - 1 file changed, 2 insertions(+), 2 deletions(-) - -ShooterIT in commit 7a35eec54: - Replace addDeferredMultiBulkLength with addReplyDeferredLen in comment - 1 file changed, 2 insertions(+), 2 deletions(-) - -Yossi Gottlieb in commit f93e1417b: - TLS: Improve tls-protocols clarity in redis.conf. - 1 file changed, 3 insertions(+), 2 deletions(-) - -ShooterIT in commit d0c9e4454: - Fix reply bytes calculation error - 1 file changed, 1 insertion(+), 1 deletion(-) - -zhaozhao.zz in commit 1cde6a060: - Tracking: flag CLIENT_TRACKING_BROKEN_REDIR when redir broken - 1 file changed, 1 insertion(+) - -Oran Agra in commit 436be3498: - fix a rare active defrag edge case bug leading to stagnation - 4 files changed, 146 insertions(+), 23 deletions(-) - -Oran Agra in commit f9d2ffdc5: - improve DEBUG MALLCTL to be able to write to write only fields. - 1 file changed, 27 insertions(+), 7 deletions(-) - -hujie in commit d7968ee92: - fix clear USER_FLAG_ALLCOMMANDS flag in acl - 1 file changed, 5 insertions(+), 4 deletions(-) - -ShooterIT in commit a902e6b25: - Redis Benchmark: generate random test data - 1 file changed, 12 insertions(+), 1 deletion(-) - -hwware in commit 9564ed7c3: - Redis-Benchmark: avoid potentical memmory leaking - 1 file changed, 1 insertion(+), 1 deletion(-) - -WuYunlong in commit 2e4182743: - Handle keys with hash tag when computing hash slot using tcl cluster client. - 1 file changed, 23 insertions(+), 2 deletions(-) - -WuYunlong in commit eb2c8b2c6: - Add a test to prove current tcl cluster client can not handle keys with hash tag. - 1 file changed, 7 insertions(+), 1 deletion(-) - -ShooterIT in commit 928e6976b: - Use dictSize to get the size of dict in dict.c - 1 file changed, 2 insertions(+), 2 deletions(-) - -Madelyn Olson in commit cdcf5af5a: - Converge hash validation for adding and removing - 1 file changed, 21 insertions(+), 14 deletions(-) - -Benjamin Sergeant in commit e8b09d220: - do not handle --cluster-yes for cluster fix mode - 1 file changed, 16 insertions(+), 7 deletions(-) - -Benjamin Sergeant in commit 57b4fb0d8: - fix typo ... - 1 file changed, 1 insertion(+), 1 deletion(-) - -Benjamin Sergeant in commit 29f25e411: - Redis-cli 6.0.1 `--cluster-yes` doesn't work (fix #7246) - 1 file changed, 5 insertions(+), 1 deletion(-) - -Oran Agra in commit 00d8b92b8: - fix valgrind test failure in replication test - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit 5e17e6276: - add regression test for the race in #7205 - 1 file changed, 52 insertions(+) - -antirez in commit 96e7c011e: - Improve the PSYNC2 test reliability. - 1 file changed, 33 insertions(+), 15 deletions(-) - -================================================================================ -Redis 6.0.3 Released Sat May 16 18:10:21 CEST 2020 -================================================================================ - -Upgrade urgency CRITICAL: a crash introduced in 6.0.2 is now fixed. - -1eab62f7e Remove the client from CLOSE_ASAP list before caching the master. - -================================================================================ -Redis 6.0.2 Released Fri May 15 22:24:36 CEST 2020 -================================================================================ - -Upgrade urgency MODERATE: many not critical bugfixes in different areas. - Critical fix to client side caching when - keys are evicted from the tracking table but - no notifications are sent. - -The following are the most serious fix: - -* XPENDING should not update consumer's seen-time -* optimize memory usage of deferred replies - fixed -* Fix CRC64 initialization outside the Redis server itself. -* stringmatchlen() should not expect null terminated strings. -* Cluster nodes availability checks improved when there is - high Pub/Sub load on the cluster bus. -* Redis Benchmark: Fix coredump because of double free -* Tracking: send eviction messages when evicting entries. -* rax.c updated from upstream antirez/rax. -* fix redis 6.0 not freeing closed connections during loading. - -New features: - -* Support setcpuaffinity on linux/bsd -* Client Side Caching: Add Tracking Prefix Number Stats in Server Info -* Add --user argument to redis-benchmark.c (ACL) - -Full list of commits: - -Yossi Gottlieb in commit 16ba33c05: - TLS: Fix test failures on recent Debian/Ubuntu. - 1 file changed, 20 deletions(-) - -Yossi Gottlieb in commit 77ae66930: - TLS: Add crypto locks for older OpenSSL support. - 1 file changed, 45 insertions(+) - -David Carlier in commit 389697988: - NetBSD build update. - 3 files changed, 30 insertions(+), 1 deletion(-) - -Madelyn Olson in commit 2435341d7: - Added a refcount on timer events to prevent deletion of recursive timer calls - 2 files changed, 12 insertions(+) - -antirez in commit 80c906bd3: - Cache master without checking of deferred close flags. - 3 files changed, 11 insertions(+), 8 deletions(-) - -antirez in commit 74249be4a: - Track events processed while blocked globally. - 5 files changed, 32 insertions(+), 17 deletions(-) - -antirez in commit 8bf660af9: - Some rework of #7234. - 4 files changed, 77 insertions(+), 65 deletions(-) - -Oran Agra in commit 9da134cd8: - fix redis 6.0 not freeing closed connections during loading. - 3 files changed, 133 insertions(+), 58 deletions(-) - -antirez in commit f7f219a13: - Regression test for #7249. - 1 file changed, 22 insertions(+) - -antirez in commit 693629585: - rax.c updated from upstream antirez/rax. - 1 file changed, 4 insertions(+), 2 deletions(-) - -antirez in commit e3b5648df: - Tracking: send eviction messages when evicting entries. - 2 files changed, 29 insertions(+), 12 deletions(-) - -Oran Agra in commit 5c41802d5: - fix unstable replication test - 1 file changed, 2 insertions(+), 2 deletions(-) - -ShooterIT in commit a23cdbb94: - Redis Benchmark: Fix coredump because of double free - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 1276058ea: - Cluster: clarify we always resolve the sender. - 1 file changed, 3 insertions(+), 1 deletion(-) - -antirez in commit 002fcde3d: - Cluster: refactor ping/data delay handling. - 1 file changed, 13 insertions(+), 11 deletions(-) - -antirez in commit 960186a71: - Cluster: introduce data_received field. - 2 files changed, 27 insertions(+), 10 deletions(-) - -antirez in commit 3672875b4: - stringmatchlen() should not expect null terminated strings. - 1 file changed, 2 insertions(+), 2 deletions(-) - -Brad Dunbar in commit 24e12641d: - Remove unreachable branch. - 1 file changed, 2 deletions(-) - -hwware in commit c7edffbd5: - add jemalloc-bg-thread config in redis conf - 1 file changed, 3 insertions(+) - -hwware in commit 8a9c84f4a: - add include guard for lolwut.h - 1 file changed, 6 insertions(+) - -antirez in commit cb683a84f: - Don't propagate spurious MULTI on DEBUG LOADAOF. - 2 files changed, 6 insertions(+), 3 deletions(-) - -antirez in commit 84d9766d6: - Dump recent backlog on master query generating errors. - 1 file changed, 29 insertions(+) - -Titouan Christophe in commit ec1e106ec: - make struct user anonymous (only typedefed) - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit e48c37316: - Test: --dont-clean should do first cleanup. - 1 file changed, 2 insertions(+), 5 deletions(-) - -Benjamin Sergeant in commit 1e561cfaa: - Add --user argument to redis-benchmark.c (ACL) - 1 file changed, 15 insertions(+), 2 deletions(-) - -antirez in commit d1af82a88: - Drop not needed part from #7194. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Muhammad Zahalqa in commit 897a360d0: - Fix compiler warnings on function rev(unsigned long) - 1 file changed, 3 insertions(+), 3 deletions(-) - -antirez in commit ac316d8cc: - Move CRC64 initialization in main(). - 2 files changed, 1 insertion(+), 4 deletions(-) - -antirez in commit fc7bc3204: - Fix CRC64 initialization outside the Redis server itself. - 1 file changed, 3 insertions(+) - -hwware in commit a6e55c096: - Client Side Caching: Add Tracking Prefix Number Stats in Server Info - 3 files changed, 8 insertions(+) - -antirez in commit b062fd523: - Fix NetBSD build by fixing redis_set_thread_title() support. - 1 file changed, 4 insertions(+), 1 deletion(-) - -antirez in commit 4efb25d9c: - Rework a bit the documentation for CPU pinning. - 2 files changed, 18 insertions(+), 8 deletions(-) - -zhenwei pi in commit d6436eb7c: - Support setcpuaffinity on linux/bsd - 12 files changed, 180 insertions(+), 1 deletion(-) - -Guy Benoish in commit 3a441c7d9: - XPENDING should not update consumer's seen-time - 4 files changed, 33 insertions(+), 20 deletions(-) - -Oran Agra in commit 75addb4fe: - optimize memory usage of deferred replies - fixed - 1 file changed, 29 insertions(+) - -Deliang Yang in commit c57d9146f: - reformat code - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit 3d3861dd8: - add daily github actions with libc malloc and valgrind - 5 files changed, 106 insertions(+), 18 deletions(-) - - -================================================================================ -Redis 6.0.1 Released Sat May 02 00:06:07 CEST 2020 -================================================================================ - -Upgrade urgency HIGH: This release fixes a crash when builiding against - Libc malloc. - -Here we revert 8110ba888, an optimization that causes a crash due to a -bug in the code. It does not happen with the default allocator because of -differences between Jemalloc and libc malloc, so this escaped all our -testing but was reported by a user. We'll add back the original optimization -that was reverted here later, after checking what happens: it is not a -critical optimization. - -The other commits are minor stuff: - -antirez in commit db73d0998: - Cast printf() argument to the format specifier. - 1 file changed, 3 insertions(+), 1 deletion(-) - -antirez in commit 7c0fe7271: - Revert "optimize memory usage of deferred replies" - 1 file changed, 31 deletions(-) - -antirez in commit 8fe25edc7: - Save a call to stopThreadedIOIfNeeded() for the base case. - 1 file changed, 3 insertions(+), 3 deletions(-) - -================================================================================ -Redis 6.0.0 GA Released Thu Apr 30 14:55:02 CEST 2020 -================================================================================ - -Upgrade urgency CRITICAL: many bugs fixed compared to the last release - candidate. Better to upgrade if you see things - affecting your environment in the changelog. - -Hi all, finally we have Redis 6.0.0 GA! Enjoy this new Redis release. -Most of the documentation was updated today so that you can likely -find what you are looking for about the new features at redis.io. -This is the list of what changed compared to the previoius release candidate: - -* XCLAIM AOF/replicas propagation fixed. -* Client side caching: new NOLOOP option to avoid getting notified about - changes performed by ourselves. -* ACL GENPASS now uses HMAC-SHA256 and have an optional "bits" argument. - It means you can use it as a general purpose "secure random strings" - primitive! -* Cluster "SLOTS" subcommand memory optimization. -* The LCS command is now a subcommand of STRALGO. -* Meaningful offset for replicas as well. More successful partial - resynchronizations. -* Optimize memory usage of deferred replies. -* Faster CRC64 algorithm for faster RDB loading. -* XINFO STREAM FULL, a new subcommand to get the whole stream state. -* CLIENT KILL USER . -* MIGRATE AUTH2 option, for ACL style authentication support. -* Other random bugfixes. - -Enjoy Redis 6! :-) -Goodbye antirez - -List of commits in this release: - -antirez in commit 1f9b82bd5: - Update help.h again before Redis 6 GA. - 1 file changed, 17 insertions(+), 12 deletions(-) - -antirez in commit 3fcffe7d0: - redis-cli: fix hints with subcommands. - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 455d8a05c: - redis-cli command help updated. - 1 file changed, 165 insertions(+), 25 deletions(-) - -zhaozhao.zz in commit 70287bbc9: - lazyfree & eviction: record latency generated by lazyfree eviction - 1 file changed, 18 insertions(+), 13 deletions(-) - -antirez in commit 7be21139a: - MIGRATE AUTH2 for ACL support. - 1 file changed, 19 insertions(+), 5 deletions(-) - -antirez in commit e1ee1a49d: - CLIENT KILL USER . - 1 file changed, 11 insertions(+) - -antirez in commit d56f058c0: - Fix tracking table max keys option in redis.conf. - 1 file changed, 12 insertions(+), 9 deletions(-) - -antirez in commit 96dd5fc93: - redis-cli: safer cluster fix with unreachalbe masters. - 1 file changed, 26 insertions(+), 1 deletion(-) - -antirez in commit 5b59d9c5d: - redis-cli: simplify cluster nodes coverage display. - 1 file changed, 10 insertions(+), 17 deletions(-) - -antirez in commit c163d4add: - redis-cli: try to make clusterManagerFixOpenSlot() more readable. - 1 file changed, 25 insertions(+), 6 deletions(-) - -Guy Benoish in commit aab74b715: - XINFO STREAM FULL should have a default COUNT of 10 - 1 file changed, 8 insertions(+), 4 deletions(-) - -antirez in commit 606134f9d: - Comment clearly why we moved some code in #6623. - 1 file changed, 4 insertions(+), 1 deletion(-) - -srzhao in commit ee627bb66: - fix pipelined WAIT performance issue. - 1 file changed, 13 insertions(+), 13 deletions(-) - -antirez in commit 47b8a7f9b: - Fix create-cluster BIN_PATH. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Guy Benoish in commit 6c0bc608a: - Extend XINFO STREAM output - 2 files changed, 226 insertions(+), 34 deletions(-) - -hwware in commit 5bfc18950: - Fix not used marco in cluster.c - 1 file changed, 1 insertion(+), 1 deletion(-) - -Itamar Haber in commit 56d628f85: - Update create-cluster - 1 file changed, 1 insertion(+), 1 deletion(-) - -Itamar Haber in commit cac9d7cf7: - Adds `BIN_PATH` to create-cluster - 1 file changed, 8 insertions(+), 6 deletions(-) - -Oran Agra in commit b712fba17: - hickup, re-fix dictEncObjKeyCompare - 1 file changed, 4 insertions(+), 4 deletions(-) - -Oran Agra in commit ea63aea72: - fix loading race in psync2 tests - 3 files changed, 15 insertions(+), 1 deletion(-) - -antirez in commit 64e588bfa: - Rework comment in dictEncObjKeyCompare(). - 1 file changed, 8 insertions(+), 9 deletions(-) - -Oran Agra in commit 0d1e8c93b: - allow dictFind using static robj - 1 file changed, 9 insertions(+), 4 deletions(-) - -Madelyn Olson in commit a1bed447b: - Added crcspeed library - 2 files changed, 341 insertions(+) - -Madelyn Olson in commit a75fa3aad: - Made crc64 test consistent - 1 file changed, 3 insertions(+), 2 deletions(-) - -Madelyn Olson in commit 52c75e9db: - Implemented CRC64 based on slice by 4 - 5 files changed, 124 insertions(+), 157 deletions(-) - -Oran Agra in commit 8110ba888: - optimize memory usage of deferred replies - 1 file changed, 31 insertions(+) - -Oran Agra in commit e4d2bb62b: - Keep track of meaningful replication offset in replicas too - 5 files changed, 212 insertions(+), 92 deletions(-) - -antirez in commit fea9788cc: - Fix STRALGO command flags. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Dave-in-lafayette in commit 2144047e1: - fix for unintended crash during panic response - 1 file changed, 1 insertion(+), 1 deletion(-) - -Guy Benoish in commit 43329c9b6: - Add the stream tag to XSETID tests - 1 file changed, 1 insertion(+), 1 deletion(-) - -Dave-in-lafayette in commit 1e17d3de7: - fix for crash during panic before all threads are up - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 3722f89f4: - LCS -> STRALGO LCS. - 4 files changed, 28 insertions(+), 15 deletions(-) - -antirez in commit 373ae6061: - Also use propagate() in streamPropagateGroupID(). - 1 file changed, 11 insertions(+), 1 deletion(-) - -yanhui13 in commit f03f1fad6: - add tcl test for cluster slots - 1 file changed, 44 insertions(+) - -yanhui13 in commit 374ffdf1c: - optimize the output of cluster slots - 1 file changed, 7 insertions(+), 4 deletions(-) - -antirez in commit 4db38d2ef: - Minor aesthetic changes to #7135. - 1 file changed, 5 insertions(+), 7 deletions(-) - -Valentino Geron in commit f0a261448: - XREADGROUP with NOACK should propagate only one XGROUP SETID command - 1 file changed, 13 insertions(+), 7 deletions(-) - -antirez in commit fbdef6a9b: - ACL: re-enable command execution of disabled users. - 1 file changed, 4 deletions(-) - -antirez in commit 05a41da75: - getRandomBytes(): use HMAC-SHA256. - 1 file changed, 30 insertions(+), 10 deletions(-) - -antirez in commit 345c3768d: - ACL GENPASS: take number of bits as argument. - 1 file changed, 21 insertions(+), 6 deletions(-) - -antirez in commit 639c8a1d9: - ACL GENPASS: emit 256 bits instead of 128. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 321acea03: - ACL: deny commands execution of disabled users. - 1 file changed, 4 insertions(+) - -Theo Buehler in commit b0920e6e8: - TLS: Fix build with SSL_OP_NO_CLIENT_RENEGOTIATION - 1 file changed, 1 insertion(+), 1 deletion(-) - -Yossi Gottlieb in commit 149b658b5: - TLS: Fix build on older verisons of OpenSSL. - 1 file changed, 2 insertions(+) - -antirez in commit 06917e581: - Tracking: test expired keys notifications. - 1 file changed, 13 insertions(+) - -antirez in commit e434b2ce4: - Tracking: NOLOOP tests. - 1 file changed, 32 insertions(+) - -antirez in commit f3a172887: - Tracking: signal key as modified when evicting. - 1 file changed, 1 insertion(+) - -antirez in commit e63bb7ec8: - Tracking: NOLOOP further implementation and fixes. - 2 files changed, 21 insertions(+), 6 deletions(-) - -antirez in commit 6791ff052: - Tracking: NOLOOP internals implementation. - 17 files changed, 174 insertions(+), 112 deletions(-) - -antirez in commit 725b8cc68: - Implement redis_set_thread_title for MacOS. - 1 file changed, 6 insertions(+) - -zhenwei pi in commit 3575b8706: - Threaded IO: set thread name for redis-server - 3 files changed, 28 insertions(+) - -antirez in commit a76c67578: - Sentinel: small refactoring of sentinelCollectTerminatedScripts(). - 1 file changed, 1 insertion(+), 2 deletions(-) - -omg-by in commit 3a27064c4: - fix(sentinel): sentinel.running_scripts will always increase more times and not reset - 1 file changed, 1 insertion(+) - -antirez in commit 5c4c73e2c: - A few comments and name changes for #7103. - 1 file changed, 13 insertions(+), 4 deletions(-) - -Oran Agra in commit 6148f9493: - testsuite run the defrag latency test solo - 3 files changed, 42 insertions(+), 2 deletions(-) - -Jamie Scott in commit 51d3012d4: - Adding acllog-max-len to Redis.conf - 1 file changed, 9 insertions(+) - -antirez in commit c39f16c42: - Fix XCLAIM propagation in AOF/replicas for blocking XREADGROUP. - 2 files changed, 8 insertions(+), 3 deletions(-) - -================================================================================ -Redis 6.0-rc4 Released Thu Apr 16 16:10:35 CEST 2020 -================================================================================ - -Upgrade urgency LOW: If you are using RC3 without issues, don't rush. - -Hi all, this the latest release candidate of Redis 6. This is likely to -be very similar to what you'll see in Redis 6 GA. Please test it and -report any issue :-) - -Main changes in this release: - - * Big INFO speedup when using a lot of of clients. - * Big speedup on all the blocking commands: now blocking - on the same key is O(1) instead of being O(N). - * Stale replicas now allow MULTI/EXEC. - * New command: LCS (Longest Common Subsequence). - * Add a new configuration to make DEL like UNLINK. - * RDB loading speedup. - * Many bugs fixed (see the commit messages at the end of this node) - -See you in 14 days for Redis 6 GA. - -List of commits: - -antirez in commit 9f594e243: - Update SDS to latest version. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 48781dd95: - RESP3: fix HELLO map len in Sentinel mode. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 371ab0cff: - Don't allow empty spaces in ACL usernames. - 1 file changed, 36 insertions(+), 8 deletions(-) - -antirez in commit b86140ac5: - Don't allow empty spaces in ACL key patterns. - 1 file changed, 12 insertions(+), 1 deletion(-) - -liumiuyong in commit a7ee3c3e7: - FIX: truncate max/min longitude,latitude related geo_point (ex: {180, 85.05112878} ) - 1 file changed, 4 insertions(+) - -Guy Benoish in commit e5b9eb817: - Typo in getTimeoutFromObjectOrReply's error reply - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 0f31bb5c1: - Fix HELLO reply in Sentinel mode, see #6160. - 1 file changed, 1 insertion(+), 1 deletion(-) - -hwware in commit b92d9a895: - fix spelling in acl.c - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit 8f896e57a: - Fix zsetAdd() top comment spelling. - 1 file changed, 3 insertions(+), 3 deletions(-) - -hayleeliu in commit 8f5157058: - fix spelling mistake in bitops.c - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit ddeda9ceb: - Fix function names in zslDeleteNode() top comment. - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit bde1f0a8e: - RESP3: change streams items from maps to arrays. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit bec68bff2: - Use the special static refcount for stack objects. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 0f239e51b: - RDB: refactor some RDB loading code into dbAddRDBLoad(). - 3 files changed, 22 insertions(+), 4 deletions(-) - -antirez in commit f855db61b: - incrRefCount(): abort on statically allocated object. - 2 files changed, 12 insertions(+), 2 deletions(-) - -antirez in commit 23094ba01: - More powerful DEBUG RELOAD. - 3 files changed, 55 insertions(+), 16 deletions(-) - -antirez in commit 8161a7a3e: - RDB: clarify a condition in rdbLoadRio(). - 2 files changed, 9 insertions(+), 2 deletions(-) - -antirez in commit 61b153073: - RDB: load files faster avoiding useless free+realloc. - 7 files changed, 40 insertions(+), 28 deletions(-) - -antirez in commit 414debfd0: - Speedup: unblock clients on keys in O(1). - 4 files changed, 50 insertions(+), 23 deletions(-) - -antirez in commit cbcd07777: - Fix ACL HELP table missing comma. - 1 file changed, 12 insertions(+), 12 deletions(-) - -mymilkbottles in commit 2437455f2: - Judge the log level in advance - 1 file changed, 1 insertion(+) - -antirez in commit 35c64b898: - Speedup INFO by counting client memory incrementally. - 4 files changed, 52 insertions(+), 26 deletions(-) - -qetu3790 in commit c3ac71748: - fix comments about RESIZE DB opcode in rdb.c - 1 file changed, 1 insertion(+), 4 deletions(-) - -antirez in commit c8dbcff9d: - Clarify redis.conf comment about lazyfree-lazy-user-del. - 1 file changed, 9 insertions(+), 5 deletions(-) - -zhaozhao.zz in commit abd5156f2: - lazyfree: add a new configuration lazyfree-lazy-user-del - 4 files changed, 7 insertions(+), 2 deletions(-) - -antirez in commit 5719b3054: - LCS: more tests. - 1 file changed, 8 insertions(+) - -antirez in commit c89e1f293: - LCS: allow KEYS / STRINGS to be anywhere. - 1 file changed, 6 deletions(-) - -antirez in commit 0b16f8d44: - LCS tests. - 1 file changed, 22 insertions(+) - -antirez in commit 9254a805d: - LCS: get rid of STOREIDX option. Fix get keys helper. - 2 files changed, 20 insertions(+), 21 deletions(-) - -antirez in commit a4c490703: - LCS: fix stale comment. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit cb92c23de: - LCS: output LCS len as well in IDX mode. - 1 file changed, 6 insertions(+), 1 deletion(-) - -antirez in commit 56a52e804: - LCS: MINMATCHLEN and WITHMATCHLEN options. - 1 file changed, 24 insertions(+), 11 deletions(-) - -antirez in commit ebb09a5c3: - LCS: 7x speedup by accessing the array with better locality. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit a9f8a8cba: - LCS: implement KEYS option. - 1 file changed, 18 insertions(+), 2 deletions(-) - -antirez in commit 4aa24e62a: - LCS: other fixes to range emission. - 1 file changed, 20 insertions(+), 16 deletions(-) - -antirez in commit 2b67b6b87: - LCS: fix emission of last range starting at index 0. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 420aac727: - LCS: implement range indexes option. - 1 file changed, 59 insertions(+), 9 deletions(-) - -antirez in commit a518a9a76: - LCS: initial functionality implemented. - 4 files changed, 156 insertions(+), 1 deletion(-) - -srzhao in commit 026cc11b0: - Check OOM at script start to get stable lua OOM state. - 3 files changed, 11 insertions(+), 4 deletions(-) - -Oran Agra in commit 02b594f6a: - diffrent fix for runtest --host --port - 2 files changed, 13 insertions(+), 13 deletions(-) - -Guy Benoish in commit f695d1830: - Try to fix time-sensitive tests in blockonkey.tcl - 1 file changed, 54 insertions(+), 1 deletion(-) - -Guy Benoish in commit 0e42cfc36: - Use __attribute__ only if __GNUC__ is defined - 1 file changed, 12 insertions(+), 3 deletions(-) - -Guy Benoish in commit 91ed9b3c4: - Modules: Perform printf-like format checks in variadic API - 1 file changed, 3 insertions(+), 3 deletions(-) - -Valentino Geron in commit 3e0d20962: - XREAD and XREADGROUP should not be allowed from scripts when BLOCK option is being used - 3 files changed, 18 insertions(+), 2 deletions(-) - -Guy Benoish in commit 240094c9b: - Stale replica should allow MULTI/EXEC - 1 file changed, 3 insertions(+), 3 deletions(-) - -Xudong Zhang in commit 209f3a1eb: - fix integer overflow - 1 file changed, 2 insertions(+), 2 deletions(-) - -Guy Benoish in commit 024c380b9: - Fix no-negative-zero test - 1 file changed, 1 insertion(+) - -Oran Agra in commit a38ff404b: - modules don't signalModifiedKey in setKey() since that's done (optionally) in RM_CloseKey - 4 files changed, 8 insertions(+), 8 deletions(-) - -Oran Agra in commit 814874d68: - change CI to build and run the module api tests - 1 file changed, 2 insertions(+) - -Oran Agra in commit 061616c1b: - fix possible warning on incomplete struct init - 1 file changed, 1 insertion(+), 1 deletion(-) - -Guy Benoish in commit 7764996be: - Make sure Redis does not reply with negative zero - 2 files changed, 10 insertions(+) - -Guy Benoish in commit eba28e2ce: - DEBUG OBJECT should pass keyname to module when loading - 3 files changed, 4 insertions(+), 4 deletions(-) - -David Carlier in commit 15c9e79a7: - debug, dump registers on arm too. - 1 file changed, 55 insertions(+), 27 deletions(-) - -hwware in commit cd2b5df97: - fix spelling in cluster.c - 1 file changed, 1 insertion(+), 1 deletion(-) - -Valentino Geron in commit 8cdc153f5: - XACK should be executed in a "all or nothing" fashion. - 2 files changed, 23 insertions(+), 1 deletion(-) - -hwware in commit b35407fa7: - add check for not switching between optin optout mode directly - 1 file changed, 12 insertions(+), 1 deletion(-) - -hwware in commit 4395889c9: - add check for not providing both optin optout flag - 1 file changed, 8 insertions(+) - -Guy Benoish in commit 1907e0f18: - PERSIST should notify a keyspace event - 1 file changed, 1 insertion(+) - -Guy Benoish in commit c35a53169: - streamReplyWithRange: Redundant XSETIDs to replica - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 6fe66e096: - Simplify comment in moduleTryServeClientBlockedOnKey(). - 1 file changed, 3 insertions(+), 12 deletions(-) - -Guy Benoish in commit 193fc241c: - Fix memory corruption in moduleHandleBlockedClients - 3 files changed, 149 insertions(+), 46 deletions(-) - -================================================================================ -Redis 6.0-rc3 Released Tue Mar 31 17:42:39 CEST 2020 -================================================================================ - -Upgrade urgency CRITICAL: A connection management bug introduced with the - SSL implementation can crash Redis easily. - -Dear users, this is a list of the major changes in this release, please check -the list of commits for detail: - -* Fix crash due to refactoring for SSL, for the connection code. -* Precise timeouts for blocking commands. Now the timeouts have HZ - resolution regardless of the number of connected clinets. New timeouts - are stored in a radix tree and sorted by expire time. -* Fix rare crash when resizing the event loop because of CONFIG maxclients. -* Fix systemd readiness after successful partial resync. -* Redis-cli ask password mode to be prompted at startup (for additional safety). -* Keyspace notifications added to MIGRATE / RESTORE. -* Threaded I/O bugs fixed. -* Implement new ACL style AUTH in Sentinel. -* Make 'requirepass' more backward compatible with Redis <= 5. -* ACL: Handle default user as disabled if it's off regardless of "nopass". -* Fix a potential inconsistency when upgrading an instance in Redis Cluster - and restarting it. The instance will act as a replica but will actually be - set as a master immediately. However the choice of what to do with already - expired keys, on loading, was made from the POV of replicas. -* Abort transactions after -READONLY error. -* Many different fixes to module APIs. -* BITFIELD_RO added to call the command on read only replicas. -* PSYNC2: meaningful offset implementation. Allow the disconnected master - that is still sending PINGs to replicas, to be able to successfully - PSYNC incrementally to new slaves, discarding the last part of the - replication backlog consisting only of PINGs. -* Fix pipelined MULTI/EXEC during Lua scripts are in BUSY state. -* Re-fix propagation API in modules, broken again after other changes. - -antirez in commit ef1b1f01: - cast raxSize() to avoid warning with format spec. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 9f347fab: - Minor changes to #7037. - 2 files changed, 14 insertions(+), 5 deletions(-) - -Guy Benoish in commit a509400d: - Modules: Test MULTI/EXEC replication of RM_Replicate - 6 files changed, 49 insertions(+), 9 deletions(-) - -Guy Benoish in commit 805c8c94: - RENAME can unblock XREADGROUP - 3 files changed, 25 insertions(+), 1 deletion(-) - -antirez in commit 97b80b57: - Fix the propagate Tcl test after module changes. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 4f6b6b80: - Modify the propagate unit test to show more cases. - 1 file changed, 30 insertions(+), 2 deletions(-) - -antirez in commit 616b1cb7: - Fix module commands propagation double MULTI bug. - 4 files changed, 25 insertions(+), 8 deletions(-) - -antirez in commit 08fdef4b: - Fix RM_Call() stale comment due to cut&paste. - 1 file changed, 1 insertion(+), 3 deletions(-) - -OMG-By in commit 26b79ca1: - fix: dict.c->dictResize()->minimal type - 1 file changed, 1 insertion(+), 1 deletion(-) - -zhaozhao.zz in commit fa418637: - PSYNC2: reset backlog_idx and master_repl_offset correctly - 1 file changed, 10 insertions(+), 5 deletions(-) - -antirez in commit bbbc80ac: - Precise timeouts: reference client pointer directly. - 1 file changed, 13 insertions(+), 16 deletions(-) - -antirez in commit c3b268a0: - timeout.c created: move client timeouts code there. - 5 files changed, 198 insertions(+), 167 deletions(-) - -Oran Agra in commit 0f7dfc37: - AOFRW on an empty stream created with MKSTREAM loads badkly - 2 files changed, 15 insertions(+), 1 deletion(-) - -antirez in commit 67643ead: - Precise timeouts: cleaup the table on unblock. - 3 files changed, 21 insertions(+), 2 deletions(-) - -antirez in commit ad94066e: - Precise timeouts: fix comments after functional change. - 2 files changed, 6 insertions(+), 6 deletions(-) - -antirez in commit a443ec2e: - Precise timeouts: use only radix tree for timeouts. - 3 files changed, 15 insertions(+), 38 deletions(-) - -antirez in commit 6862fd70: - Precise timeouts: fast exit for clientsHandleShortTimeout(). - 1 file changed, 1 insertion(+) - -antirez in commit 30f1df8c: - Precise timeouts: fix bugs in initial implementation. - 2 files changed, 5 insertions(+), 1 deletion(-) - -antirez in commit 7add0f24: - Precise timeouts: working initial implementation. - 3 files changed, 110 insertions(+), 28 deletions(-) - -antirez in commit 9d6d1779: - Precise timeouts: refactor unblocking on timeout. - 2 files changed, 33 insertions(+), 13 deletions(-) - -antirez in commit 316a8f15: - PSYNC2: fix backlog_idx when adjusting for meaningful offset - 1 file changed, 3 insertions(+) - -伯成 in commit 11db53f8: - Boost up performance for redis PUB-SUB patterns matching - 3 files changed, 43 insertions(+), 11 deletions(-) - -antirez in commit e257f121: - PSYNC2: meaningful offset test. - 2 files changed, 62 insertions(+) - -antirez in commit 5f72f696: - PSYNC2: meaningful offset implemented. - 3 files changed, 40 insertions(+), 1 deletion(-) - -antirez in commit 8caa2714: - Explain why we allow transactions in -BUSY state. - 1 file changed, 9 insertions(+), 2 deletions(-) - -Oran Agra in commit e43cd831: - MULTI/EXEC during LUA script timeout are messed up - 2 files changed, 73 insertions(+) - -antirez in commit 34b89832: - Improve comments of replicationCacheMasterUsingMyself(). - 1 file changed, 6 insertions(+), 1 deletion(-) - -antirez in commit 70a98a43: - Fix BITFIELD_RO test. - 2 files changed, 5 insertions(+), 5 deletions(-) - -antirez in commit 8783304a: - Abort transactions after -READONLY error. Fix #7014. - 1 file changed, 1 insertion(+) - -antirez in commit ec9cf002: - Minor changes to BITFIELD_RO PR #6951. - 1 file changed, 9 insertions(+), 6 deletions(-) - -bodong.ybd in commit b3e4abf0: - Added BITFIELD_RO variants for read-only operations. - 4 files changed, 54 insertions(+), 1 deletion(-) - -antirez in commit 50f8f950: - Modules: updated function doc after #7003. - 1 file changed, 6 insertions(+), 1 deletion(-) - -Guy Benoish in commit f2f3dc5e: - Allow RM_GetContextFlags to work with ctx==NULL - 1 file changed, 16 insertions(+), 14 deletions(-) - -hwware in commit eb808879: - fix potentical memory leak in redis-cli - 1 file changed, 2 insertions(+) - -Yossi Gottlieb in commit cdcab0e8: - Fix crashes related to failed/rejected accepts. - 1 file changed, 6 insertions(+), 5 deletions(-) - -Yossi Gottlieb in commit 50dcd9f9: - Cluster: fix misleading accept errors. - 1 file changed, 4 insertions(+), 3 deletions(-) - -Yossi Gottlieb in commit 87dbd8f5: - Conns: Fix connClose() / connAccept() behavior. - 3 files changed, 48 insertions(+), 32 deletions(-) - -hwware in commit 81e8686c: - remove redundant Semicolon - 1 file changed, 1 insertion(+), 1 deletion(-) - -hwware in commit c7524a7e: - clean CLIENT_TRACKING_CACHING flag when disabled caching - 1 file changed, 1 insertion(+), 1 deletion(-) - -hwware in commit 2dd1ca6a: - add missing commands in cluster help - 1 file changed, 2 insertions(+), 1 deletion(-) - -artix in commit 95324b81: - Support Redis Cluster Proxy PROXY INFO command - 1 file changed, 5 insertions(+), 1 deletion(-) - -박승현 in commit 04c53fa1: - Update redis.conf - 1 file changed, 1 insertion(+), 1 deletion(-) - -WuYunlong in commit 0578157d: - Fix master replica inconsistency for upgrading scenario. - 3 files changed, 9 insertions(+), 2 deletions(-) - -WuYunlong in commit 299f1d02: - Add 14-consistency-check.tcl to prove there is a data consistency issue. - 1 file changed, 87 insertions(+) - -antirez in commit 61b98f32: - Regression test for #7011. - 1 file changed, 7 insertions(+) - -antirez in commit 34ea2f4e: - ACL: default user off should not allow automatic authentication. - 2 files changed, 3 insertions(+), 2 deletions(-) - -antirez in commit cbbf9b39: - Sentinel: document auth-user directive. - 1 file changed, 12 insertions(+) - -antirez in commit 9c2e42dd: - ACL: Make Redis 6 more backward compatible with requirepass. - 4 files changed, 17 insertions(+), 15 deletions(-) - -antirez in commit d387f67d: - Sentinel: implement auth-user directive for ACLs. - 1 file changed, 38 insertions(+), 7 deletions(-) - -zhaozhao.zz in commit 7c078416: - Threaded IO: bugfix client kill may crash redis - 1 file changed, 11 insertions(+), 5 deletions(-) - -zhaozhao.zz in commit 9cc7038e: - Threaded IO: handle pending reads clients ASAP after event loop - 1 file changed, 3 insertions(+), 1 deletion(-) - -antirez in commit da8c7c49: - Example sentinel conf: document requirepass. - 1 file changed, 8 insertions(+) - -antirez in commit bdb338cf: - Aesthetic changes in PR #6989. - 1 file changed, 9 insertions(+), 5 deletions(-) - -zhaozhao.zz in commit b3e03054: - Threaded IO: bugfix #6988 process events while blocked - 1 file changed, 5 insertions(+) - -antirez in commit e628f944: - Restore newline at the end of redis-cli.c - 1 file changed, 2 insertions(+), 1 deletion(-) - -chendianqiang in commit 5d4c4df3: - use correct list for moduleUnregisterUsedAPI - 1 file changed, 1 insertion(+), 1 deletion(-) - -guodongxiaren in commit da14982d: - string literal should be const char* - 1 file changed, 1 insertion(+), 1 deletion(-) - -Itamar Haber in commit dc8885a1: - Adds keyspace notifications to migrate and restore - 1 file changed, 3 insertions(+), 1 deletion(-) - -bodong.ybd in commit bfb18e55: - Remove duplicate obj files in Makefile - 1 file changed, 2 insertions(+), 2 deletions(-) - -bodong.ybd in commit 76d57161: - Fix bug of tcl test using external server - 2 files changed, 8 insertions(+), 2 deletions(-) - -fengpf in commit 0e5820d8: - fix comments in latency.c - 2 files changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 916dd79f: - Update linenoise. - 1 file changed, 2 insertions(+), 1 deletion(-) - -lifubang in commit c0c67c9b: - add askpass mode - 1 file changed, 19 insertions(+), 1 deletion(-) - -lifubang in commit e1c29434: - update linenoise to https://github.com/antirez/linenoise/tree/fc9667a81d43911a6690fb1e68c16e6e3bb8df05 - 4 files changed, 59 insertions(+), 4 deletions(-) - -Jamie Scott in commit e5a063bc: - Remove default guidance in Redis.conf - 1 file changed, 1 insertion(+), 2 deletions(-) - -Jamie Scott in commit d28cbaf7: - Update Redis.conf to improve TLS usability - 1 file changed, 2 insertions(+), 1 deletion(-) - -Johannes Truschnigg in commit 23d5e8b8: - Signal systemd readiness atfer Partial Resync - 1 file changed, 4 insertions(+) - -Oran Agra in commit 61738154: - fix for flaky psync2 test - 1 file changed, 21 insertions(+) - -antirez in commit 70e0e499: - ae.c: fix crash when resizing the event loop. - 1 file changed, 6 insertions(+), 2 deletions(-) - -antirez in commit b3e4aa67: - Fix release notes spelling mistake. - 1 file changed, 1 insertion(+), 1 deletion(-) - - -================================================================================ -Redis 6.0 RC2 Released Thu Mar 05 15:40:53 CET 2020 -================================================================================ - -Upgrade urgency MODERATE: Normal bugfixing release of a non-GA branch. - -Hi Redis users, Redis 6 is approaching and will be released 30th of April. -New release candidates will be released at the end of March, then another -one mid April, to finally reach the GA at the end of April. - -Redis 6 RC2 brings many fixes and new things, especially in the area of -client side caching. This is the list of big changes in this release. As -usually you can find the full list of commits at the end: - -New features and improvements: - -* ACL LOG: log denied commands, keys accesses and authentications. -* Client side caching redesigned. Now we use keys not caching slots. -* Client side caching: Broadcasting mode implemented. -* Client side caching: OPTIN/OPTOUT modes implemented. -* Remove RDB files used for replication in persistence-less instances (option). - -Fixes (only selected ones, see commits for all the fixes): - -* Different fixes to streams in edge cases. -* Fix duplicated CLIENT SETNAME reply because of RESP3 changes. -* Fix crash due to new active expire division by zero. -* Avoid sentinel changes promoted_slave to be its own replica. -* Fix bug on KEYS command where pattern starts with * followed by \x00. -* Threaded I/O: now the main thread is used as well to do I/O. -* Many fixes to modules APIs, and more to come in the next RCs. -* ld2string should fail if string contains \0 in the middle. -* Make the Redis test more reliable. -* Fix SPOP returning nil (see #4709). WARNING: API change. - -qetu3790 in commit 4af0d7fd: - Fix not used constant in lru_test_mode. - 1 file changed, 1 insertion(+), 1 deletion(-) - -hwware in commit 6ef01878: - add missing file marco - 1 file changed, 5 insertions(+) - -ShooterIT in commit fe81d5c8: - Avoid compiler warnings - 1 file changed, 1 insertion(+) - -antirez in commit c2f01d7f: - RDB deletion: document it in example redis.conf. - 1 file changed, 13 insertions(+) - -antirez in commit 127e09bc: - Make sync RDB deletion configurable. Default to no. - 3 files changed, 22 insertions(+), 4 deletions(-) - -antirez in commit a20303c6: - Check that the file exists in removeRDBUsedToSyncReplicas(). - 1 file changed, 8 insertions(+), 4 deletions(-) - -antirez in commit 7a23b945: - Log RDB deletion in persistence-less instances. - 1 file changed, 15 insertions(+), 2 deletions(-) - -antirez in commit baaf869f: - Introduce bg_unlink(). - 1 file changed, 31 insertions(+), 3 deletions(-) - -antirez in commit be4bc1a5: - Remove RDB files used for replication in persistence-less instances. - 3 files changed, 56 insertions(+), 1 deletion(-) - -antirez in commit 07dc1b42: - Use a smaller getkeys global buffer. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit 10e71b3d: - Optimize temporary memory allocations for getKeysFromCommand mechanism - 1 file changed, 31 insertions(+), 10 deletions(-) - -antirez in commit edc0ed14: - Modules: reformat RM_Scan() top comment a bit. - 1 file changed, 21 insertions(+), 12 deletions(-) - -antirez in commit c5319612: - Modules: more details in RM_Scan API top comment. - 1 file changed, 22 insertions(+), 6 deletions(-) - -Oran Agra in commit fff6b26a: - RM_Scan disable dict rehashing - 2 files changed, 21 insertions(+), 6 deletions(-) - -Guy Benoish in commit 65048460: - Add RM_CreateStringFromDouble - 2 files changed, 14 insertions(+) - -Oran Agra in commit 3144a278: - add no_auth to COMMAND INFO - 1 file changed, 1 insertion(+) - -Oran Agra in commit afe0b16c: - module api docs for aux_save and aux_load - 2 files changed, 7 insertions(+), 1 deletion(-) - -Guy Benoish in commit df152b0c: - streamReplyWithRangeFromConsumerPEL: Redundant streamDecodeID - 1 file changed, 1 insertion(+), 3 deletions(-) - -antirez in commit e3c1f439: - Show Redis version when not understanding a config directive. - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 141c0679: - Changelog: explain Redis 6 SPOP change. - 1 file changed, 4 insertions(+), 1 deletion(-) - -bodong.ybd in commit fe902461: - Fix spop return nil #4709 - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 9d4219eb: - Fix SDS misuse in enumConfigSet(). Related to #6778. - 1 file changed, 3 insertions(+), 3 deletions(-) - -antirez in commit 84243064: - Remove useless comment from enumConfigSet(). - 1 file changed, 1 deletion(-) - -Ponnuvel Palaniyappan in commit dafb94db: - Fix a potential overflow with strncpy - 1 file changed, 5 insertions(+), 5 deletions(-) - -antirez in commit ea697b63: - Improve aeDeleteEventLoop() top comment grammar. - 1 file changed, 2 insertions(+), 1 deletion(-) - -wangyuan21 in commit dd479880: - free time event when delete eventloop - 1 file changed, 7 insertions(+) - -srzhao in commit ecf3b2ef: - fix impl of aof-child whitelist SIGUSR1 feature. - 1 file changed, 5 insertions(+), 4 deletions(-) - -meir@redislabs.com in commit 2966132c: - Changed log level for module fork api from 'notice' to 'verbos'. - 1 file changed, 2 insertions(+), 2 deletions(-) - -hwware in commit 7277e5d8: - format fix - 1 file changed, 1 insertion(+), 1 deletion(-) - -hwware in commit 1bb5ee9c: - fix potentical memory leaks - 1 file changed, 4 insertions(+), 1 deletion(-) - -Hengjian Tang in commit 97329733: - modify the read buf size according to the write buf size PROTO_IOBUF_LEN defined before - 1 file changed, 1 insertion(+), 1 deletion(-) - -Ariel in commit 15ea1324: - fix ThreadSafeContext lock/unlock function names - 1 file changed, 2 insertions(+), 2 deletions(-) - -Guy Benoish in commit 4d12c37c: - XREADGROUP should propagate XCALIM/SETID in MULTI/EXEC - 1 file changed, 2 insertions(+), 2 deletions(-) - -Oran Agra in commit 12626ce9: - fix race in module api test for fork - 2 files changed, 2 insertions(+), 3 deletions(-) - -Guy Benoish in commit 2ecab0b6: - Modules: Do not auto-unblock clients if not blocked on keys - 1 file changed, 22 insertions(+), 7 deletions(-) - -Oran Agra in commit 635321d4: - fix github actions failing latency test for active defrag - part 2 - 2 files changed, 5 insertions(+), 4 deletions(-) - -Oran Agra in commit 0b988fa9: - fix github actions failing latency test for active defrag - 2 files changed, 14 insertions(+), 13 deletions(-) - -Oran Agra in commit 60096bc1: - Fix latency sensitivity of new defrag test - 1 file changed, 32 insertions(+), 8 deletions(-) - -antirez in commit b4395426: - Tracking: optin/out implemented. - 3 files changed, 82 insertions(+), 16 deletions(-) - -antirez in commit ef3551d1: - Test engine: experimental change to avoid busy port problems. - 1 file changed, 84 insertions(+), 49 deletions(-) - -antirez in commit 72c05351: - Test engine: detect timeout when checking for Redis startup. - 1 file changed, 11 insertions(+), 1 deletion(-) - -antirez in commit 294c9af4: - Test engine: better tracking of what workers are doing. - 2 files changed, 12 insertions(+), 4 deletions(-) - -hwware in commit ba027079: - add missing subcommand description for debug oom - 1 file changed, 1 insertion(+) - -Guy Benoish in commit 5d0890c0: - Fix memory leak in test_ld_conv - 1 file changed, 4 insertions(+) - -Madelyn Olson in commit d1f22eac: - Give an error message if you specify redirect twice - 1 file changed, 7 insertions(+) - -Madelyn Olson in commit 762fbcb6: - Minor CSC fixes and fixed documentation - 2 files changed, 16 insertions(+), 17 deletions(-) - -Oran Agra in commit 349aa245: - Defrag big lists in portions to avoid latency and freeze - 4 files changed, 350 insertions(+), 34 deletions(-) - -Guy Benoish in commit b4ddc7b7: - XGROUP DESTROY should unblock XREADGROUP with -NOGROUP - 2 files changed, 11 insertions(+) - -hayashier in commit 73806f74: - fix typo from fss to rss - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit b6129f86: - Test is more complex now, increase default timeout. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit f15fb727: - Tracking: fix max-keys configuration directive. - 2 files changed, 2 insertions(+), 2 deletions(-) - -Itamar Haber in commit e374573f: - Fixes segfault on calling trackingGetTotalKeys - 1 file changed, 1 insertion(+) - -antirez in commit 73d47d57: - Signal key as modified when expired on-access. - 1 file changed, 4 insertions(+), 2 deletions(-) - -antirez in commit b7cb28d5: - Tracking: first set of tests for the feature. - 1 file changed, 66 insertions(+) - -antirez in commit 1db72571: - Tracking: fix operators precedence error in bcast check. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit fe96e29d: - Tracking: fix behavior when switchinig from normal to BCAST. - 1 file changed, 11 insertions(+), 1 deletion(-) - -antirez in commit f21be1ec: - Tracking: fix sending messages bug + tracking off bug. - 2 files changed, 28 insertions(+), 20 deletions(-) - -antirez in commit 6fb1aa23: - Tracking: BCAST: basic feature now works. - 3 files changed, 55 insertions(+), 40 deletions(-) - -antirez in commit d4fe79a1: - Tracking: BCAST: broadcasting of keys in prefixes implemented. - 2 files changed, 94 insertions(+), 9 deletions(-) - -antirez in commit abb81c63: - Tracking: BCAST: registration in the prefix table. - 3 files changed, 67 insertions(+), 20 deletions(-) - -antirez in commit 77da9608: - Tracking: BCAST: parsing of the options + skeleton. - 4 files changed, 73 insertions(+), 19 deletions(-) - -antirez in commit 3e8c69a9: - Tracking: always reply with an array of keys. - 2 files changed, 10 insertions(+), 3 deletions(-) - -antirez in commit a788c373: - Tracking: minor change of names and new INFO field. - 4 files changed, 11 insertions(+), 4 deletions(-) - -antirez in commit df838927: - Rax.c: populate data field after random walk. - 1 file changed, 1 insertion(+) - -antirez in commit 0517da36: - Tracking: rename INFO field with total items. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 3c16d6b3: - Tracking: first conversion from hashing to key names. - 3 files changed, 84 insertions(+), 114 deletions(-) - -Oran Agra in commit 3b4f1477: - add no-slowlog option to RM_CreateCommand - 1 file changed, 3 insertions(+) - -Khem Raj in commit 5e762d84: - Mark extern definition of SDS_NOINIT in sds.h - 1 file changed, 1 insertion(+), 1 deletion(-) - -lifubang in commit 54f5499a: - correct help info for --user and --pass - 1 file changed, 2 insertions(+), 2 deletions(-) - -Seunghoon Woo in commit 0c952b13: - [FIX] revisit CVE-2015-8080 vulnerability - 1 file changed, 6 insertions(+), 4 deletions(-) - -Guy Benoish in commit dd34f703: - Diskless-load emptyDb-related fixes - 3 files changed, 44 insertions(+), 28 deletions(-) - -lifubang in commit 5e042dbc: - fix ssl flag check for redis-cli - 1 file changed, 10 insertions(+), 9 deletions(-) - -Guy Benoish in commit dcbe8bfa: - Exclude "keymiss" notification from NOTIFY_ALL - 5 files changed, 12 insertions(+), 7 deletions(-) - -Oran Agra in commit 36caf2e4: - update RM_SignalModifiedKey doc comment - 1 file changed, 2 insertions(+), 1 deletion(-) - -Oran Agra in commit 3067352a: - Add handling of short read of module id in rdb - 1 file changed, 4 insertions(+), 1 deletion(-) - -Yossi Gottlieb in commit 9baaf858: - TLS: Update documentation. - 2 files changed, 32 insertions(+), 31 deletions(-) - -Oran Agra in commit 4440133e: - A few non-data commands that should be allowed while loading or stale - 1 file changed, 8 insertions(+), 8 deletions(-) - -Oran Agra in commit c9577941: - Memory leak when bind config is provided twice - 1 file changed, 4 insertions(+) - -Oran Agra in commit 1333a46b: - fix maxmemory config warning - 1 file changed, 3 insertions(+), 2 deletions(-) - -Oran Agra in commit 8e7282eb: - Fix client flags to be int64 in module.c - 1 file changed, 3 insertions(+), 3 deletions(-) - -Oran Agra in commit a678390e: - moduleRDBLoadError, add key name, and use panic rather than exit - 1 file changed, 5 insertions(+), 4 deletions(-) - -Oran Agra in commit 919fbf42: - reduce repeated calls to use_diskless_load - 1 file changed, 3 insertions(+), 4 deletions(-) - -Oran Agra in commit 22e45d46: - freeClientAsync don't lock mutex if there's just one thread - 1 file changed, 6 insertions(+), 1 deletion(-) - -Oran Agra in commit ba289244: - move restartAOFAfterSYNC from replicaofCommand to replicationUnsetMaster - 1 file changed, 4 insertions(+), 3 deletions(-) - -Oran Agra in commit f42ce57d: - stopAppendOnly resets aof_rewrite_scheduled - 1 file changed, 1 insertion(+) - -Oran Agra in commit df096bc9: - add SAVE subcommand to ACL HELP and top comment - 1 file changed, 2 insertions(+) - -Oran Agra in commit a55e5847: - DEBUG HELP - add PROTOCOL - 1 file changed, 3 insertions(+), 2 deletions(-) - -Guy Benoish in commit 5a6cfbf4: - Some refactroing using getClientType instead of CLIENT_SLAVE - 2 files changed, 18 insertions(+), 26 deletions(-) - -Guy Benoish in commit fae306b3: - Fix small bugs related to replica and monitor ambiguity - 2 files changed, 8 insertions(+), 6 deletions(-) - -Yossi Gottlieb in commit 73630966: - TLS: Some redis.conf clarifications. - 1 file changed, 10 insertions(+), 11 deletions(-) - -Oran Agra in commit 488e1947: - config.c verbose error replies for CONFIG SET, like config file parsing - 1 file changed, 31 insertions(+), 97 deletions(-) - -Oran Agra in commit c82ccf06: - memoryGetKeys helper function so that ACL can limit access to keys for MEMORY command - 3 files changed, 18 insertions(+), 1 deletion(-) - -antirez in commit 51c1a9f8: - ACL LOG: make max log entries configurable. - 4 files changed, 19 insertions(+) - -antirez in commit ea1e1b12: - ACL LOG: test for AUTH reason. - 1 file changed, 9 insertions(+) - -antirez in commit 7379c78a: - ACL LOG: log failed auth attempts. - 5 files changed, 34 insertions(+), 12 deletions(-) - -antirez in commit 9f6e84f6: - ACL LOG: implement a few basic tests. - 1 file changed, 87 insertions(+) - -antirez in commit 82790e51: - ACL LOG: also log ACL errors in the scripting/MULTI ctx. - 2 files changed, 6 insertions(+), 2 deletions(-) - -antirez in commit 943008eb: - ACL LOG: implement LOG RESET. - 1 file changed, 6 insertions(+), 2 deletions(-) - -antirez in commit e271a611: - ACL LOG: group similar entries in a given time delta. - 1 file changed, 58 insertions(+), 3 deletions(-) - -antirez in commit f1974d5d: - ACL LOG: actually emit entries. - 3 files changed, 34 insertions(+), 5 deletions(-) - -antirez in commit d9b153c9: - ACL LOG: implement ACL LOG subcommadn skeleton. - 1 file changed, 37 insertions(+) - -antirez in commit 577fc438: - ACL LOG: data structures and initial functions. - 5 files changed, 54 insertions(+), 5 deletions(-) - -Leo Murillo in commit f7a94526: - Set ZSKIPLIST_MAXLEVEL to optimal value given 2^64 elements and p=0.25 - 1 file changed, 1 insertion(+), 1 deletion(-) - -WuYunlong in commit eecfa979: - Fix lua related memory leak. - 1 file changed, 1 insertion(+) - -WuYunlong in commit d2509811: - Add tcl regression test in scripting.tcl to reproduce memory leak. - 1 file changed, 5 insertions(+) - -Yossi Gottlieb in commit 29d4a150: - TLS: Fix missing initialization in redis-cli. - 1 file changed, 9 insertions(+) - -Oran Agra in commit ec0c61da: - fix uninitialized info_cb var in module.c - 1 file changed, 1 insertion(+) - -Guy Benoish in commit 6fe55c2f: - ld2string should fail if string contains \0 in the middle - 5 files changed, 20 insertions(+), 11 deletions(-) - -antirez in commit bbce3ba9: - Add more info in the unblockClientFromModule() function. - 1 file changed, 7 insertions(+), 1 deletion(-) - -Guy Benoish in commit 40295fb3: - Modules: Fix blocked-client-related memory leak - 3 files changed, 51 insertions(+), 6 deletions(-) - -antirez in commit 8e9d19bc: - Change error message for #6775. - 1 file changed, 2 insertions(+), 2 deletions(-) - -Vasyl Melnychuk in commit ba146d4c: - Make error when submitting command in incorrect context more explicit - 1 file changed, 4 insertions(+), 1 deletion(-) - -antirez in commit 721a39dd: - Document I/O threads in redis.conf. - 1 file changed, 46 insertions(+) - -antirez in commit 5be3a15a: - Setting N I/O threads should mean N-1 additional + 1 main thread. - 1 file changed, 25 insertions(+), 22 deletions(-) - -antirez in commit cbabf779: - Simplify #6379 changes. - 2 files changed, 4 insertions(+), 9 deletions(-) - -WuYunlong in commit 658749cc: - Free allocated sds in pfdebugCommand() to avoid memory leak. - 1 file changed, 1 insertion(+) - -WuYunlong in commit 47988c96: - Fix potential memory leak of clusterLoadConfig(). - 1 file changed, 20 insertions(+), 5 deletions(-) - -WuYunlong in commit cc90f79b: - Fix potential memory leak of rioWriteBulkStreamID(). - 1 file changed, 4 insertions(+), 1 deletion(-) - -antirez in commit ecd17e81: - Jump to right label on AOF parsing error. - 1 file changed, 6 insertions(+), 4 deletions(-) - -antirez in commit 1927932b: - Port PR #6110 to new connection object code. - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit f2df5773: - A few comments about main thread serving I/O as well. - 1 file changed, 7 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit b3ff8a4b: - Threaded IO: use main thread to handle read work - 1 file changed, 8 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit b1f2c510: - Threaded IO: use main thread to handle write work - 1 file changed, 10 insertions(+), 2 deletions(-) - -ShooterIT in commit 7bbafc56: - Rename rdb asynchronously - 1 file changed, 7 insertions(+) - -Leo Murillo in commit c7f75266: - Fix bug on KEYS command where pattern starts with * followed by \x00 (null char). - 1 file changed, 1 insertion(+), 1 deletion(-) - -Jamie Scott in commit ed7ea13a: - Update to directive in redis.conf (missing s) - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 3be77623: - Free fakeclient argv on AOF error. - 1 file changed, 11 insertions(+), 3 deletions(-) - -antirez in commit 15f6b748: - Git ignore: ignore more files. - 1 file changed, 2 insertions(+) - -Guy Benoish in commit 1b5bf40c: - Blocking XREAD[GROUP] should always reply with valid data (or timeout) - 3 files changed, 44 insertions(+), 10 deletions(-) - -John Sully in commit 954c20ed: - Add support for incremental build with header files - 2 files changed, 6 insertions(+), 1 deletion(-) - -WuYunlong in commit 11c3afd7: - Fix petential cluster link error. - 1 file changed, 4 insertions(+) - -Yossi Gottlieb in commit b752e83d: - Add REDISMODULE_CTX_FLAGS_MULTI_DIRTY. - 2 files changed, 8 insertions(+) - -hwware in commit e16eb874: - typo fix in acl.c - 1 file changed, 2 insertions(+), 2 deletions(-) - -Itamar Haber in commit 35ea9d23: - Adjusts 'io_threads_num' max to 128 - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 38729126: - XCLAIM: Create the consumer only on successful claims. - 1 file changed, 4 insertions(+), 2 deletions(-) - -yz1509 in commit b9a15303: - avoid sentinel changes promoted_slave to be its own replica. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 5e7e5e6b: - Fix active expire division by zero. - 1 file changed, 7 insertions(+), 4 deletions(-) - -antirez in commit e61dde88: - Fix duplicated CLIENT SETNAME reply. - 1 file changed, 1 deletion(-) - -Guy Benoish in commit cddf1da2: - Stream: Handle streamID-related edge cases - 4 files changed, 54 insertions(+), 4 deletions(-) - -Oran Agra in commit 52ea44e5: - config.c adjust config limits and mutable - 2 files changed, 7 insertions(+), 7 deletions(-) - -antirez in commit 0f28ea16: - Inline protocol: handle empty strings well. - 1 file changed, 2 insertions(+), 6 deletions(-) - -antirez in commit 00e5fefe: - Fix ip and missing mode in RM_GetClusterNodeInfo(). - 1 file changed, 5 insertions(+), 2 deletions(-) - -================================================================================ -Redis 6.0 RC1 Released Thu Dec 19 09:58:24 CEST 2019 -================================================================================ - -Upgrade urgency LOW: This is the first RC of Redis 6. - -Introduction to the Redis 6 release -=================================== - -Redis 6 improves Redis in a number of key areas and is one of the largest -Redis releases in the history of the project, so here we'll list only -the biggest features in this release: - -* The modules system now has a number of new APIs that allow module authors - to make things otherwise not possible in the past. It is possible to - store arbitrary module private data in RDB files, to hook on different - server events, capture and rewrite commands executions, block clients on - keys, and so forth. -* The Redis active expire cycle was rewritten for much faster eviction of keys - that are already expired. Now the effort is tunable. -* Redis now supports SSL on all channels. -* ACL support, you can define users that can run only certain commands and/or - can only access only certain keys patterns. -* Redis now supports a new protocol called RESP3, which returns more - semantical replies: new clients using this protocol can understand just - from the reply what type to return to the calling program. -* There is server-side support for client-side caching of key values. This - feature is still experimental and will get more changes during the next - release candidates, but you can already test it and read about it here: - https://redis.io/topics/client-side-caching -* Redis can now optionally use threads to handle I/O, allowing to serve - 2 times as much operations per second in a single instance when - pipelining cannot be used. -* Diskless replication is now supported even on replicas: a replica is now - able, under certain conditions the user can configure, to load the RDB - in the first synchronization directly from the socket to the memory. -* Redis-benchmark now supports a Redis Cluster mode. -* SRANDMEMBER and similar commands have a better distribution. -* Redis-cli improvements. -* Systemd support rewritten. -* A Redis Cluster proxy was released here: - https://github.com/artix75/redis-cluster-proxy -* A Disque module for Redis was released here: - https://github.com/antirez/disque-module - -Thanks to all the users and developers who made this release possible. -We'll follow up with more RC releases, until the code looks production ready -and we don't get reports of serious issues for a while. - -A special thank you for the amount of work put into this release -(in decreasing number of commits, only listing contributors with more -than a single commit) by: - - 685 antirez - 81 zhaozhao.zz - 76 Oran Agra - 51 artix - 28 Madelyn Olson - 27 Yossi Gottlieb - 15 David Carlier - 14 Guy Benoish - 14 Guy Korland - 13 Itamar Haber - 9 Angus Pearson - 8 WuYunlong - 8 yongman - 7 vattezhang - 7 Chris Lamb - 5 Dvir Volk - 5 meir@redislabs.com - 5 chendianqiang - 5 John Sully - 4 dejun.xdj - 4 Daniel Dai - 4 Johannes Truschnigg - 4 swilly22 - 3 Bruce Merry - 3 filipecosta90 - 3 youjiali1995 - 2 James Rouzier - 2 Andrey Bugaevskiy - 2 Brad Solomon - 2 Hamid Alaei - 2 Michael Chaten - 2 Steve Webster - 2 Wander Hillen - 2 Weiliang Li - 2 Yuan Zhou - 2 charsyam - 2 hujie - 2 jem - 2 shenlongxing - 2 valentino - 2 zhudacai 00228490 - 2 喜欢兰花山丘 - -Migrating from 5.0 to 6.0 -========================= - -Redis 6.0 is mostly a strict superset of 5.0, you should not have any problem -upgrading your application from 5.0 to 6.0. However this is a list of small -non-backward compatible changes introduced in the 6.0 release: - -* The SPOP command no longer returns null when the set key does not - exist. Now it returns the empty set as it should and as happens when it is - called with a 0 argument. This is technically a fix, however it changes the - old behavior. - --------------------------------------------------------------------------------- - -Credits: For each release, a list of changes with the relative author is -provided. Where not specified the implementation and design is done by -Salvatore Sanfilippo. Thanks to Redis Labs for making all this possible. -Also many thanks to all the other contributors and the amazing community -we have. - -Commit messages may contain additional credits. - -Enjoy, -Salvatore diff --git a/CONDUCT b/CONDUCT new file mode 100644 index 000000000..e0e15e268 --- /dev/null +++ b/CONDUCT @@ -0,0 +1,96 @@ +Contributor Covenant Code of Conduct +Our Pledge +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. +Our Standards +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, +and learning from the experience +* Focusing on what is best not just for us as individuals, but for the +overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or +advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others’ private information, such as a physical or email +address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a +professional setting + +Enforcement Responsibilities +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. +Scope +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. +Enforcement +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +this email address: redis@redis.io. +All complaints will be reviewed and investigated promptly and fairly. +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. +Enforcement Guidelines +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: +1. Correction +Community Impact: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. +Consequence: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. +2. Warning +Community Impact: A violation through a single incident or series +of actions. +Consequence: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. +3. Temporary Ban +Community Impact: A serious violation of community standards, including +sustained inappropriate behavior. +Consequence: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. +4. Permanent Ban +Community Impact: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. +Consequence: A permanent ban from any sort of public interaction within +the community. +Attribution +This Code of Conduct is adapted from the Contributor Covenant, +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. +Community Impact Guidelines were inspired by Mozilla’s code of conduct +enforcement ladder. +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. \ No newline at end of file diff --git a/COPYING b/COPYING index e1e5dd654..d2a5dd51a 100644 --- a/COPYING +++ b/COPYING @@ -1,5 +1,6 @@ Copyright (c) 2006-2020, Salvatore Sanfilippo -Copyright (C) 2019-2020, John Sully +Copyright (C) 2019-2021, John Sully +Copyright (C) 2020-2021, EQ Alpha Technology Ltd. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/README.md b/README.md index ac8aa60d3..9e7506e64 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,13 @@ installed): % ./runtest --tls +If TLS is built, running the tests with TLS enabled (you will need `tcl-tls` +installed): + + % ./utils/gen-test-certs.sh + % ./runtest --tls + + Fixing build problems with dependencies or cached build options --------- @@ -169,6 +176,18 @@ To compile against jemalloc on Mac OS X systems, use: % make MALLOC=jemalloc +Monotonic clock +--------------- + +By default, Redis will build using the POSIX clock_gettime function as the +monotonic clock source. On most modern systems, the internal processor clock +can be used to improve performance. Cautions can be found here: + http://oliveryang.net/2015/09/pitfalls-of-TSC-usage/ + +To build with support for the processor's internal instruction clock, use: + + % make CFLAGS="-DUSE_PROCESSOR_CLOCK" + Verbose build ------------- diff --git a/deps/Makefile b/deps/Makefile index f4fff9f2e..39ee1dc92 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -37,6 +37,7 @@ distclean: -(cd linenoise && $(MAKE) clean) > /dev/null || true -(cd lua && $(MAKE) clean) > /dev/null || true -(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true + -(cd hdr_histogram && $(MAKE) clean) > /dev/null || true -(rm -f .make-*) .PHONY: distclean @@ -62,18 +63,24 @@ memkind: cd memkind && $(MAKE) +hdr_histogram: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd hdr_histogram && $(MAKE) + +.PHONY: hdr_histogram + ifeq ($(uname_S),SunOS) # Make isinf() available LUA_CFLAGS= -D__C99FEATURES__=1 endif -LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' $(CFLAGS) +LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' -DLUA_USE_MKSTEMP $(CFLAGS) LUA_LDFLAGS+= $(LDFLAGS) # lua's Makefile defines AR="ar rcu", which is unusual, and makes it more # challenging to cross-compile lua (and redis). These defines make it easier # to fit redis into cross-compilation environments, which typically set AR. AR=ar -ARFLAGS=rcu +ARFLAGS=rc lua: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) @@ -86,7 +93,7 @@ JEMALLOC_LDFLAGS= $(LDFLAGS) jemalloc: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd jemalloc && ./configure --with-version=5.2.1-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --enable-cc-silence CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" + cd jemalloc && ./configure --with-version=5.2.1-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" cd jemalloc && $(MAKE) CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" lib/libjemalloc.a .PHONY: jemalloc diff --git a/deps/README.md b/deps/README.md index 02c99052f..c3ec424b6 100644 --- a/deps/README.md +++ b/deps/README.md @@ -17,7 +17,7 @@ active defragmentation logic. However this feature of Redis is not mandatory and Redis is able to understand if the Jemalloc version it is compiled against supports such Redis-specific modifications. So in theory, if you are not interested in the active defragmentation, you can replace Jemalloc -just following tose steps: +just following these steps: 1. Remove the jemalloc directory. 2. Substitute it with the new jemalloc source tree. diff --git a/deps/hdr_histogram/COPYING.txt b/deps/hdr_histogram/COPYING.txt new file mode 100644 index 000000000..0e259d42c --- /dev/null +++ b/deps/hdr_histogram/COPYING.txt @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/deps/hdr_histogram/LICENSE.txt b/deps/hdr_histogram/LICENSE.txt new file mode 100644 index 000000000..9b4e66ed7 --- /dev/null +++ b/deps/hdr_histogram/LICENSE.txt @@ -0,0 +1,41 @@ +The code in this repository code was Written by Gil Tene, Michael Barker, +and Matt Warren, and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +For users of this code who wish to consume it under the "BSD" license +rather than under the public domain or CC0 contribution text mentioned +above, the code found under this directory is *also* provided under the +following license (commonly referred to as the BSD 2-Clause License). This +license does not detract from the above stated release of the code into +the public domain, and simply represents an additional license granted by +the Author. + +----------------------------------------------------------------------------- +** Beginning of "BSD 2-Clause License" text. ** + + Copyright (c) 2012, 2013, 2014 Gil Tene + Copyright (c) 2014 Michael Barker + Copyright (c) 2014 Matt Warren + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. diff --git a/deps/hdr_histogram/Makefile b/deps/hdr_histogram/Makefile new file mode 100644 index 000000000..83d3760ae --- /dev/null +++ b/deps/hdr_histogram/Makefile @@ -0,0 +1,20 @@ +STD= +WARN= -Wall +OPT= -Os + +R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) +R_LDFLAGS= $(LDFLAGS) +DEBUG= -g + +R_CC=$(CC) $(R_CFLAGS) +R_LD=$(CC) $(R_LDFLAGS) + +hdr_histogram.o: hdr_histogram.h hdr_histogram.c + +.c.o: + $(R_CC) -c $< + +clean: + rm -f *.o + + diff --git a/deps/hdr_histogram/README.md b/deps/hdr_histogram/README.md new file mode 100644 index 000000000..5f62c234c --- /dev/null +++ b/deps/hdr_histogram/README.md @@ -0,0 +1,10 @@ +HdrHistogram_c v0.11.0 + +---------------------------------------------- + +This port contains a subset of the 'C' version of High Dynamic Range (HDR) Histogram available at [github.com/HdrHistogram/HdrHistogram_c](https://github.com/HdrHistogram/HdrHistogram_c). + + +The code present on `hdr_histogram.c`, `hdr_histogram.h`, and `hdr_atomic.c` was Written by Gil Tene, Michael Barker, +and Matt Warren, and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/. \ No newline at end of file diff --git a/deps/hdr_histogram/hdr_atomic.h b/deps/hdr_histogram/hdr_atomic.h new file mode 100644 index 000000000..ae1056a83 --- /dev/null +++ b/deps/hdr_histogram/hdr_atomic.h @@ -0,0 +1,146 @@ +/** + * hdr_atomic.h + * Written by Philip Orwig and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#ifndef HDR_ATOMIC_H__ +#define HDR_ATOMIC_H__ + + +#if defined(_MSC_VER) + +#include +#include +#include + +static void __inline * hdr_atomic_load_pointer(void** pointer) +{ + _ReadBarrier(); + return *pointer; +} + +static void hdr_atomic_store_pointer(void** pointer, void* value) +{ + _WriteBarrier(); + *pointer = value; +} + +static int64_t __inline hdr_atomic_load_64(int64_t* field) +{ + _ReadBarrier(); + return *field; +} + +static void __inline hdr_atomic_store_64(int64_t* field, int64_t value) +{ + _WriteBarrier(); + *field = value; +} + +static int64_t __inline hdr_atomic_exchange_64(volatile int64_t* field, int64_t value) +{ +#if defined(_WIN64) + return _InterlockedExchange64(field, value); +#else + int64_t comparand; + int64_t initial_value = *field; + do + { + comparand = initial_value; + initial_value = _InterlockedCompareExchange64(field, value, comparand); + } + while (comparand != initial_value); + + return initial_value; +#endif +} + +static int64_t __inline hdr_atomic_add_fetch_64(volatile int64_t* field, int64_t value) +{ +#if defined(_WIN64) + return _InterlockedExchangeAdd64(field, value) + value; +#else + int64_t comparand; + int64_t initial_value = *field; + do + { + comparand = initial_value; + initial_value = _InterlockedCompareExchange64(field, comparand + value, comparand); + } + while (comparand != initial_value); + + return initial_value + value; +#endif +} + +static bool __inline hdr_atomic_compare_exchange_64(volatile int64_t* field, int64_t* expected, int64_t desired) +{ + return *expected == _InterlockedCompareExchange64(field, desired, *expected); +} + +#elif defined(__ATOMIC_SEQ_CST) + +#define hdr_atomic_load_pointer(x) __atomic_load_n(x, __ATOMIC_SEQ_CST) +#define hdr_atomic_store_pointer(f,v) __atomic_store_n(f,v, __ATOMIC_SEQ_CST) +#define hdr_atomic_load_64(x) __atomic_load_n(x, __ATOMIC_SEQ_CST) +#define hdr_atomic_store_64(f,v) __atomic_store_n(f,v, __ATOMIC_SEQ_CST) +#define hdr_atomic_exchange_64(f,i) __atomic_exchange_n(f,i, __ATOMIC_SEQ_CST) +#define hdr_atomic_add_fetch_64(field, value) __atomic_add_fetch(field, value, __ATOMIC_SEQ_CST) +#define hdr_atomic_compare_exchange_64(field, expected, desired) __atomic_compare_exchange_n(field, expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) + +#elif defined(__x86_64__) + +#include +#include + +static inline void* hdr_atomic_load_pointer(void** pointer) +{ + void* p = *pointer; + asm volatile ("" ::: "memory"); + return p; +} + +static inline void hdr_atomic_store_pointer(void** pointer, void* value) +{ + asm volatile ("lock; xchgq %0, %1" : "+q" (value), "+m" (*pointer)); +} + +static inline int64_t hdr_atomic_load_64(int64_t* field) +{ + int64_t i = *field; + asm volatile ("" ::: "memory"); + return i; +} + +static inline void hdr_atomic_store_64(int64_t* field, int64_t value) +{ + asm volatile ("lock; xchgq %0, %1" : "+q" (value), "+m" (*field)); +} + +static inline int64_t hdr_atomic_exchange_64(volatile int64_t* field, int64_t value) +{ + int64_t result = 0; + asm volatile ("lock; xchgq %1, %2" : "=r" (result), "+q" (value), "+m" (*field)); + return result; +} + +static inline int64_t hdr_atomic_add_fetch_64(volatile int64_t* field, int64_t value) +{ + return __sync_add_and_fetch(field, value); +} + +static inline bool hdr_atomic_compare_exchange_64(volatile int64_t* field, int64_t* expected, int64_t desired) +{ + int64_t original; + asm volatile( "lock; cmpxchgq %2, %1" : "=a"(original), "+m"(*field) : "q"(desired), "0"(*expected)); + return original == *expected; +} + +#else + +#error "Unable to determine atomic operations for your platform" + +#endif + +#endif /* HDR_ATOMIC_H__ */ diff --git a/deps/hdr_histogram/hdr_histogram.c b/deps/hdr_histogram/hdr_histogram.c new file mode 100644 index 000000000..5f5286f2f --- /dev/null +++ b/deps/hdr_histogram/hdr_histogram.c @@ -0,0 +1,1155 @@ +/** + * hdr_histogram.c + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hdr_histogram.h" +#include "hdr_atomic.h" + +/* ###### ####### ## ## ## ## ######## ###### */ +/* ## ## ## ## ## ## ### ## ## ## ## */ +/* ## ## ## ## ## #### ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ###### */ +/* ## ## ## ## ## ## #### ## ## */ +/* ## ## ## ## ## ## ## ### ## ## ## */ +/* ###### ####### ####### ## ## ## ###### */ + +static int32_t normalize_index(const struct hdr_histogram* h, int32_t index) +{ + int32_t normalized_index; + int32_t adjustment = 0; + if (h->normalizing_index_offset == 0) + { + return index; + } + + normalized_index = index - h->normalizing_index_offset; + + if (normalized_index < 0) + { + adjustment = h->counts_len; + } + else if (normalized_index >= h->counts_len) + { + adjustment = -h->counts_len; + } + + return normalized_index + adjustment; +} + +static int64_t counts_get_direct(const struct hdr_histogram* h, int32_t index) +{ + return h->counts[index]; +} + +static int64_t counts_get_normalised(const struct hdr_histogram* h, int32_t index) +{ + return counts_get_direct(h, normalize_index(h, index)); +} + +static void counts_inc_normalised( + struct hdr_histogram* h, int32_t index, int64_t value) +{ + int32_t normalised_index = normalize_index(h, index); + h->counts[normalised_index] += value; + h->total_count += value; +} + +static void counts_inc_normalised_atomic( + struct hdr_histogram* h, int32_t index, int64_t value) +{ + int32_t normalised_index = normalize_index(h, index); + + hdr_atomic_add_fetch_64(&h->counts[normalised_index], value); + hdr_atomic_add_fetch_64(&h->total_count, value); +} + +static void update_min_max(struct hdr_histogram* h, int64_t value) +{ + h->min_value = (value < h->min_value && value != 0) ? value : h->min_value; + h->max_value = (value > h->max_value) ? value : h->max_value; +} + +static void update_min_max_atomic(struct hdr_histogram* h, int64_t value) +{ + int64_t current_min_value; + int64_t current_max_value; + do + { + current_min_value = hdr_atomic_load_64(&h->min_value); + + if (0 == value || current_min_value <= value) + { + break; + } + } + while (!hdr_atomic_compare_exchange_64(&h->min_value, ¤t_min_value, value)); + + do + { + current_max_value = hdr_atomic_load_64(&h->max_value); + + if (value <= current_max_value) + { + break; + } + } + while (!hdr_atomic_compare_exchange_64(&h->max_value, ¤t_max_value, value)); +} + + +/* ## ## ######## #### ## #### ######## ## ## */ +/* ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## #### */ +/* ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## */ +/* ####### ## #### ######## #### ## ## */ + +static int64_t power(int64_t base, int64_t exp) +{ + int64_t result = 1; + while(exp) + { + result *= base; exp--; + } + return result; +} + +#if defined(_MSC_VER) +# if defined(_WIN64) +# pragma intrinsic(_BitScanReverse64) +# else +# pragma intrinsic(_BitScanReverse) +# endif +#endif + +static int32_t count_leading_zeros_64(int64_t value) +{ +#if defined(_MSC_VER) + uint32_t leading_zero = 0; +#if defined(_WIN64) + _BitScanReverse64(&leading_zero, value); +#else + uint32_t high = value >> 32; + if (_BitScanReverse(&leading_zero, high)) + { + leading_zero += 32; + } + else + { + uint32_t low = value & 0x00000000FFFFFFFF; + _BitScanReverse(&leading_zero, low); + } +#endif + return 63 - leading_zero; /* smallest power of 2 containing value */ +#else + return __builtin_clzll(value); /* smallest power of 2 containing value */ +#endif +} + +static int32_t get_bucket_index(const struct hdr_histogram* h, int64_t value) +{ + int32_t pow2ceiling = 64 - count_leading_zeros_64(value | h->sub_bucket_mask); /* smallest power of 2 containing value */ + return pow2ceiling - h->unit_magnitude - (h->sub_bucket_half_count_magnitude + 1); +} + +static int32_t get_sub_bucket_index(int64_t value, int32_t bucket_index, int32_t unit_magnitude) +{ + return (int32_t)(value >> (bucket_index + unit_magnitude)); +} + +static int32_t counts_index(const struct hdr_histogram* h, int32_t bucket_index, int32_t sub_bucket_index) +{ + /* Calculate the index for the first entry in the bucket: */ + /* (The following is the equivalent of ((bucket_index + 1) * subBucketHalfCount) ): */ + int32_t bucket_base_index = (bucket_index + 1) << h->sub_bucket_half_count_magnitude; + /* Calculate the offset in the bucket: */ + int32_t offset_in_bucket = sub_bucket_index - h->sub_bucket_half_count; + /* The following is the equivalent of ((sub_bucket_index - subBucketHalfCount) + bucketBaseIndex; */ + return bucket_base_index + offset_in_bucket; +} + +static int64_t value_from_index(int32_t bucket_index, int32_t sub_bucket_index, int32_t unit_magnitude) +{ + return ((int64_t) sub_bucket_index) << (bucket_index + unit_magnitude); +} + +int32_t counts_index_for(const struct hdr_histogram* h, int64_t value) +{ + int32_t bucket_index = get_bucket_index(h, value); + int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude); + + return counts_index(h, bucket_index, sub_bucket_index); +} + +int64_t hdr_value_at_index(const struct hdr_histogram *h, int32_t index) +{ + int32_t bucket_index = (index >> h->sub_bucket_half_count_magnitude) - 1; + int32_t sub_bucket_index = (index & (h->sub_bucket_half_count - 1)) + h->sub_bucket_half_count; + + if (bucket_index < 0) + { + sub_bucket_index -= h->sub_bucket_half_count; + bucket_index = 0; + } + + return value_from_index(bucket_index, sub_bucket_index, h->unit_magnitude); +} + +int64_t hdr_size_of_equivalent_value_range(const struct hdr_histogram* h, int64_t value) +{ + int32_t bucket_index = get_bucket_index(h, value); + int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude); + int32_t adjusted_bucket = (sub_bucket_index >= h->sub_bucket_count) ? (bucket_index + 1) : bucket_index; + return INT64_C(1) << (h->unit_magnitude + adjusted_bucket); +} + +static int64_t lowest_equivalent_value(const struct hdr_histogram* h, int64_t value) +{ + int32_t bucket_index = get_bucket_index(h, value); + int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude); + return value_from_index(bucket_index, sub_bucket_index, h->unit_magnitude); +} + +int64_t hdr_next_non_equivalent_value(const struct hdr_histogram *h, int64_t value) +{ + return lowest_equivalent_value(h, value) + hdr_size_of_equivalent_value_range(h, value); +} + +static int64_t highest_equivalent_value(const struct hdr_histogram* h, int64_t value) +{ + return hdr_next_non_equivalent_value(h, value) - 1; +} + +int64_t hdr_median_equivalent_value(const struct hdr_histogram *h, int64_t value) +{ + return lowest_equivalent_value(h, value) + (hdr_size_of_equivalent_value_range(h, value) >> 1); +} + +static int64_t non_zero_min(const struct hdr_histogram* h) +{ + if (INT64_MAX == h->min_value) + { + return INT64_MAX; + } + + return lowest_equivalent_value(h, h->min_value); +} + +void hdr_reset_internal_counters(struct hdr_histogram* h) +{ + int min_non_zero_index = -1; + int max_index = -1; + int64_t observed_total_count = 0; + int i; + + for (i = 0; i < h->counts_len; i++) + { + int64_t count_at_index; + + if ((count_at_index = counts_get_direct(h, i)) > 0) + { + observed_total_count += count_at_index; + max_index = i; + if (min_non_zero_index == -1 && i != 0) + { + min_non_zero_index = i; + } + } + } + + if (max_index == -1) + { + h->max_value = 0; + } + else + { + int64_t max_value = hdr_value_at_index(h, max_index); + h->max_value = highest_equivalent_value(h, max_value); + } + + if (min_non_zero_index == -1) + { + h->min_value = INT64_MAX; + } + else + { + h->min_value = hdr_value_at_index(h, min_non_zero_index); + } + + h->total_count = observed_total_count; +} + +static int32_t buckets_needed_to_cover_value(int64_t value, int32_t sub_bucket_count, int32_t unit_magnitude) +{ + int64_t smallest_untrackable_value = ((int64_t) sub_bucket_count) << unit_magnitude; + int32_t buckets_needed = 1; + while (smallest_untrackable_value <= value) + { + if (smallest_untrackable_value > INT64_MAX / 2) + { + return buckets_needed + 1; + } + smallest_untrackable_value <<= 1; + buckets_needed++; + } + + return buckets_needed; +} + +/* ## ## ######## ## ## ####### ######## ## ## */ +/* ### ### ## ### ### ## ## ## ## ## ## */ +/* #### #### ## #### #### ## ## ## ## #### */ +/* ## ### ## ###### ## ### ## ## ## ######## ## */ +/* ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ######## ## ## ####### ## ## ## */ + +int hdr_calculate_bucket_config( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram_bucket_config* cfg) +{ + int32_t sub_bucket_count_magnitude; + int64_t largest_value_with_single_unit_resolution; + + if (lowest_trackable_value < 1 || + significant_figures < 1 || 5 < significant_figures || + lowest_trackable_value * 2 > highest_trackable_value) + { + return EINVAL; + } + + cfg->lowest_trackable_value = lowest_trackable_value; + cfg->significant_figures = significant_figures; + cfg->highest_trackable_value = highest_trackable_value; + + largest_value_with_single_unit_resolution = 2 * power(10, significant_figures); + sub_bucket_count_magnitude = (int32_t) ceil(log((double)largest_value_with_single_unit_resolution) / log(2)); + cfg->sub_bucket_half_count_magnitude = ((sub_bucket_count_magnitude > 1) ? sub_bucket_count_magnitude : 1) - 1; + + cfg->unit_magnitude = (int32_t) floor(log((double)lowest_trackable_value) / log(2)); + + cfg->sub_bucket_count = (int32_t) pow(2, (cfg->sub_bucket_half_count_magnitude + 1)); + cfg->sub_bucket_half_count = cfg->sub_bucket_count / 2; + cfg->sub_bucket_mask = ((int64_t) cfg->sub_bucket_count - 1) << cfg->unit_magnitude; + + if (cfg->unit_magnitude + cfg->sub_bucket_half_count_magnitude > 61) + { + return EINVAL; + } + + cfg->bucket_count = buckets_needed_to_cover_value(highest_trackable_value, cfg->sub_bucket_count, (int32_t)cfg->unit_magnitude); + cfg->counts_len = (cfg->bucket_count + 1) * (cfg->sub_bucket_count / 2); + + return 0; +} + +void hdr_init_preallocated(struct hdr_histogram* h, struct hdr_histogram_bucket_config* cfg) +{ + h->lowest_trackable_value = cfg->lowest_trackable_value; + h->highest_trackable_value = cfg->highest_trackable_value; + h->unit_magnitude = (int32_t)cfg->unit_magnitude; + h->significant_figures = (int32_t)cfg->significant_figures; + h->sub_bucket_half_count_magnitude = cfg->sub_bucket_half_count_magnitude; + h->sub_bucket_half_count = cfg->sub_bucket_half_count; + h->sub_bucket_mask = cfg->sub_bucket_mask; + h->sub_bucket_count = cfg->sub_bucket_count; + h->min_value = INT64_MAX; + h->max_value = 0; + h->normalizing_index_offset = 0; + h->conversion_ratio = 1.0; + h->bucket_count = cfg->bucket_count; + h->counts_len = cfg->counts_len; + h->total_count = 0; +} + +int hdr_init( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram** result) +{ + int64_t* counts; + struct hdr_histogram_bucket_config cfg; + struct hdr_histogram* histogram; + + int r = hdr_calculate_bucket_config(lowest_trackable_value, highest_trackable_value, significant_figures, &cfg); + if (r) + { + return r; + } + + counts = (int64_t*) calloc((size_t) cfg.counts_len, sizeof(int64_t)); + if (!counts) + { + return ENOMEM; + } + + histogram = (struct hdr_histogram*) calloc(1, sizeof(struct hdr_histogram)); + if (!histogram) + { + free(counts); + return ENOMEM; + } + + histogram->counts = counts; + + hdr_init_preallocated(histogram, &cfg); + *result = histogram; + + return 0; +} + +void hdr_close(struct hdr_histogram* h) +{ + if (h) { + free(h->counts); + free(h); + } +} + +int hdr_alloc(int64_t highest_trackable_value, int significant_figures, struct hdr_histogram** result) +{ + return hdr_init(1, highest_trackable_value, significant_figures, result); +} + +/* reset a histogram to zero. */ +void hdr_reset(struct hdr_histogram *h) +{ + h->total_count=0; + h->min_value = INT64_MAX; + h->max_value = 0; + memset(h->counts, 0, (sizeof(int64_t) * h->counts_len)); +} + +size_t hdr_get_memory_size(struct hdr_histogram *h) +{ + return sizeof(struct hdr_histogram) + h->counts_len * sizeof(int64_t); +} + +/* ## ## ######## ######## ### ######## ######## ###### */ +/* ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ######## ## ## ## ## ## ###### ###### */ +/* ## ## ## ## ## ######### ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## */ +/* ####### ## ######## ## ## ## ######## ###### */ + + +bool hdr_record_value(struct hdr_histogram* h, int64_t value) +{ + return hdr_record_values(h, value, 1); +} + +bool hdr_record_value_atomic(struct hdr_histogram* h, int64_t value) +{ + return hdr_record_values_atomic(h, value, 1); +} + +bool hdr_record_values(struct hdr_histogram* h, int64_t value, int64_t count) +{ + int32_t counts_index; + + if (value < 0) + { + return false; + } + + counts_index = counts_index_for(h, value); + + if (counts_index < 0 || h->counts_len <= counts_index) + { + return false; + } + + counts_inc_normalised(h, counts_index, count); + update_min_max(h, value); + + return true; +} + +bool hdr_record_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count) +{ + int32_t counts_index; + + if (value < 0) + { + return false; + } + + counts_index = counts_index_for(h, value); + + if (counts_index < 0 || h->counts_len <= counts_index) + { + return false; + } + + counts_inc_normalised_atomic(h, counts_index, count); + update_min_max_atomic(h, value); + + return true; +} + +bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t expected_interval) +{ + return hdr_record_corrected_values(h, value, 1, expected_interval); +} + +bool hdr_record_corrected_value_atomic(struct hdr_histogram* h, int64_t value, int64_t expected_interval) +{ + return hdr_record_corrected_values_atomic(h, value, 1, expected_interval); +} + +bool hdr_record_corrected_values(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval) +{ + int64_t missing_value; + + if (!hdr_record_values(h, value, count)) + { + return false; + } + + if (expected_interval <= 0 || value <= expected_interval) + { + return true; + } + + missing_value = value - expected_interval; + for (; missing_value >= expected_interval; missing_value -= expected_interval) + { + if (!hdr_record_values(h, missing_value, count)) + { + return false; + } + } + + return true; +} + +bool hdr_record_corrected_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval) +{ + int64_t missing_value; + + if (!hdr_record_values_atomic(h, value, count)) + { + return false; + } + + if (expected_interval <= 0 || value <= expected_interval) + { + return true; + } + + missing_value = value - expected_interval; + for (; missing_value >= expected_interval; missing_value -= expected_interval) + { + if (!hdr_record_values_atomic(h, missing_value, count)) + { + return false; + } + } + + return true; +} + +int64_t hdr_add(struct hdr_histogram* h, const struct hdr_histogram* from) +{ + struct hdr_iter iter; + int64_t dropped = 0; + hdr_iter_recorded_init(&iter, from); + + while (hdr_iter_next(&iter)) + { + int64_t value = iter.value; + int64_t count = iter.count; + + if (!hdr_record_values(h, value, count)) + { + dropped += count; + } + } + + return dropped; +} + +int64_t hdr_add_while_correcting_for_coordinated_omission( + struct hdr_histogram* h, struct hdr_histogram* from, int64_t expected_interval) +{ + struct hdr_iter iter; + int64_t dropped = 0; + hdr_iter_recorded_init(&iter, from); + + while (hdr_iter_next(&iter)) + { + int64_t value = iter.value; + int64_t count = iter.count; + + if (!hdr_record_corrected_values(h, value, count, expected_interval)) + { + dropped += count; + } + } + + return dropped; +} + + + +/* ## ## ### ## ## ## ######## ###### */ +/* ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ###### ###### */ +/* ## ## ######### ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## */ +/* ### ## ## ######## ####### ######## ###### */ + + +int64_t hdr_max(const struct hdr_histogram* h) +{ + if (0 == h->max_value) + { + return 0; + } + + return highest_equivalent_value(h, h->max_value); +} + +int64_t hdr_min(const struct hdr_histogram* h) +{ + if (0 < hdr_count_at_index(h, 0)) + { + return 0; + } + + return non_zero_min(h); +} + +int64_t hdr_value_at_percentile(const struct hdr_histogram* h, double percentile) +{ + struct hdr_iter iter; + int64_t total = 0; + double requested_percentile = percentile < 100.0 ? percentile : 100.0; + int64_t count_at_percentile = + (int64_t) (((requested_percentile / 100) * h->total_count) + 0.5); + count_at_percentile = count_at_percentile > 1 ? count_at_percentile : 1; + + hdr_iter_init(&iter, h); + + while (hdr_iter_next(&iter)) + { + total += iter.count; + + if (total >= count_at_percentile) + { + int64_t value_from_index = iter.value; + return highest_equivalent_value(h, value_from_index); + } + } + + return 0; +} + +double hdr_mean(const struct hdr_histogram* h) +{ + struct hdr_iter iter; + int64_t total = 0; + + hdr_iter_init(&iter, h); + + while (hdr_iter_next(&iter)) + { + if (0 != iter.count) + { + total += iter.count * hdr_median_equivalent_value(h, iter.value); + } + } + + return (total * 1.0) / h->total_count; +} + +double hdr_stddev(const struct hdr_histogram* h) +{ + double mean = hdr_mean(h); + double geometric_dev_total = 0.0; + + struct hdr_iter iter; + hdr_iter_init(&iter, h); + + while (hdr_iter_next(&iter)) + { + if (0 != iter.count) + { + double dev = (hdr_median_equivalent_value(h, iter.value) * 1.0) - mean; + geometric_dev_total += (dev * dev) * iter.count; + } + } + + return sqrt(geometric_dev_total / h->total_count); +} + +bool hdr_values_are_equivalent(const struct hdr_histogram* h, int64_t a, int64_t b) +{ + return lowest_equivalent_value(h, a) == lowest_equivalent_value(h, b); +} + +int64_t hdr_lowest_equivalent_value(const struct hdr_histogram* h, int64_t value) +{ + return lowest_equivalent_value(h, value); +} + +int64_t hdr_count_at_value(const struct hdr_histogram* h, int64_t value) +{ + return counts_get_normalised(h, counts_index_for(h, value)); +} + +int64_t hdr_count_at_index(const struct hdr_histogram* h, int32_t index) +{ + return counts_get_normalised(h, index); +} + + +/* #### ######## ######## ######## ### ######## ####### ######## ###### */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ###### ######## ## ## ## ## ## ######## ###### */ +/* ## ## ## ## ## ######### ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* #### ## ######## ## ## ## ## ## ####### ## ## ###### */ + + +static bool has_buckets(struct hdr_iter* iter) +{ + return iter->counts_index < iter->h->counts_len; +} + +static bool has_next(struct hdr_iter* iter) +{ + return iter->cumulative_count < iter->total_count; +} + +static bool move_next(struct hdr_iter* iter) +{ + iter->counts_index++; + + if (!has_buckets(iter)) + { + return false; + } + + iter->count = counts_get_normalised(iter->h, iter->counts_index); + iter->cumulative_count += iter->count; + + iter->value = hdr_value_at_index(iter->h, iter->counts_index); + iter->highest_equivalent_value = highest_equivalent_value(iter->h, iter->value); + iter->lowest_equivalent_value = lowest_equivalent_value(iter->h, iter->value); + iter->median_equivalent_value = hdr_median_equivalent_value(iter->h, iter->value); + + return true; +} + +static int64_t peek_next_value_from_index(struct hdr_iter* iter) +{ + return hdr_value_at_index(iter->h, iter->counts_index + 1); +} + +static bool next_value_greater_than_reporting_level_upper_bound( + struct hdr_iter *iter, int64_t reporting_level_upper_bound) +{ + if (iter->counts_index >= iter->h->counts_len) + { + return false; + } + + return peek_next_value_from_index(iter) > reporting_level_upper_bound; +} + +static bool basic_iter_next(struct hdr_iter *iter) +{ + if (!has_next(iter) || iter->counts_index >= iter->h->counts_len) + { + return false; + } + + move_next(iter); + + return true; +} + +static void update_iterated_values(struct hdr_iter* iter, int64_t new_value_iterated_to) +{ + iter->value_iterated_from = iter->value_iterated_to; + iter->value_iterated_to = new_value_iterated_to; +} + +static bool all_values_iter_next(struct hdr_iter* iter) +{ + bool result = move_next(iter); + + if (result) + { + update_iterated_values(iter, iter->value); + } + + return result; +} + +void hdr_iter_init(struct hdr_iter* iter, const struct hdr_histogram* h) +{ + iter->h = h; + + iter->counts_index = -1; + iter->total_count = h->total_count; + iter->count = 0; + iter->cumulative_count = 0; + iter->value = 0; + iter->highest_equivalent_value = 0; + iter->value_iterated_from = 0; + iter->value_iterated_to = 0; + + iter->_next_fp = all_values_iter_next; +} + +bool hdr_iter_next(struct hdr_iter* iter) +{ + return iter->_next_fp(iter); +} + +/* ######## ######## ######## ###### ######## ## ## ######## #### ## ######## ###### */ +/* ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## #### ## ## ## ## ## ## */ +/* ######## ###### ######## ## ###### ## ## ## ## ## ## ###### ###### */ +/* ## ## ## ## ## ## ## #### ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## */ +/* ## ######## ## ## ###### ######## ## ## ## #### ######## ######## ###### */ + +static bool percentile_iter_next(struct hdr_iter* iter) +{ + int64_t temp, half_distance, percentile_reporting_ticks; + + struct hdr_iter_percentiles* percentiles = &iter->specifics.percentiles; + + if (!has_next(iter)) + { + if (percentiles->seen_last_value) + { + return false; + } + + percentiles->seen_last_value = true; + percentiles->percentile = 100.0; + + return true; + } + + if (iter->counts_index == -1 && !basic_iter_next(iter)) + { + return false; + } + + do + { + double current_percentile = (100.0 * (double) iter->cumulative_count) / iter->h->total_count; + if (iter->count != 0 && + percentiles->percentile_to_iterate_to <= current_percentile) + { + update_iterated_values(iter, highest_equivalent_value(iter->h, iter->value)); + + percentiles->percentile = percentiles->percentile_to_iterate_to; + temp = (int64_t)(log(100 / (100.0 - (percentiles->percentile_to_iterate_to))) / log(2)) + 1; + half_distance = (int64_t) pow(2, (double) temp); + percentile_reporting_ticks = percentiles->ticks_per_half_distance * half_distance; + percentiles->percentile_to_iterate_to += 100.0 / percentile_reporting_ticks; + + return true; + } + } + while (basic_iter_next(iter)); + + return true; +} + +void hdr_iter_percentile_init(struct hdr_iter* iter, const struct hdr_histogram* h, int32_t ticks_per_half_distance) +{ + iter->h = h; + + hdr_iter_init(iter, h); + + iter->specifics.percentiles.seen_last_value = false; + iter->specifics.percentiles.ticks_per_half_distance = ticks_per_half_distance; + iter->specifics.percentiles.percentile_to_iterate_to = 0.0; + iter->specifics.percentiles.percentile = 0.0; + + iter->_next_fp = percentile_iter_next; +} + +static void format_line_string(char* str, size_t len, int significant_figures, format_type format) +{ +#if defined(_MSC_VER) +#define snprintf _snprintf +#pragma warning(push) +#pragma warning(disable: 4996) +#endif + const char* format_str = "%s%d%s"; + + switch (format) + { + case CSV: + snprintf(str, len, format_str, "%.", significant_figures, "f,%f,%d,%.2f\n"); + break; + case CLASSIC: + snprintf(str, len, format_str, "%12.", significant_figures, "f %12f %12d %12.2f\n"); + break; + default: + snprintf(str, len, format_str, "%12.", significant_figures, "f %12f %12d %12.2f\n"); + } +#if defined(_MSC_VER) +#undef snprintf +#pragma warning(pop) +#endif +} + + +/* ######## ######## ###### ####### ######## ######## ######## ######## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ######## ###### ## ## ## ######## ## ## ###### ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ######## ###### ####### ## ## ######## ######## ######## */ + + +static bool recorded_iter_next(struct hdr_iter* iter) +{ + while (basic_iter_next(iter)) + { + if (iter->count != 0) + { + update_iterated_values(iter, iter->value); + + iter->specifics.recorded.count_added_in_this_iteration_step = iter->count; + return true; + } + } + + return false; +} + +void hdr_iter_recorded_init(struct hdr_iter* iter, const struct hdr_histogram* h) +{ + hdr_iter_init(iter, h); + + iter->specifics.recorded.count_added_in_this_iteration_step = 0; + + iter->_next_fp = recorded_iter_next; +} + +/* ## #### ## ## ######## ### ######## */ +/* ## ## ### ## ## ## ## ## ## */ +/* ## ## #### ## ## ## ## ## ## */ +/* ## ## ## ## ## ###### ## ## ######## */ +/* ## ## ## #### ## ######### ## ## */ +/* ## ## ## ### ## ## ## ## ## */ +/* ######## #### ## ## ######## ## ## ## ## */ + + +static bool iter_linear_next(struct hdr_iter* iter) +{ + struct hdr_iter_linear* linear = &iter->specifics.linear; + + linear->count_added_in_this_iteration_step = 0; + + if (has_next(iter) || + next_value_greater_than_reporting_level_upper_bound( + iter, linear->next_value_reporting_level_lowest_equivalent)) + { + do + { + if (iter->value >= linear->next_value_reporting_level_lowest_equivalent) + { + update_iterated_values(iter, linear->next_value_reporting_level); + + linear->next_value_reporting_level += linear->value_units_per_bucket; + linear->next_value_reporting_level_lowest_equivalent = + lowest_equivalent_value(iter->h, linear->next_value_reporting_level); + + return true; + } + + if (!move_next(iter)) + { + return true; + } + + linear->count_added_in_this_iteration_step += iter->count; + } + while (true); + } + + return false; +} + + +void hdr_iter_linear_init(struct hdr_iter* iter, const struct hdr_histogram* h, int64_t value_units_per_bucket) +{ + hdr_iter_init(iter, h); + + iter->specifics.linear.count_added_in_this_iteration_step = 0; + iter->specifics.linear.value_units_per_bucket = value_units_per_bucket; + iter->specifics.linear.next_value_reporting_level = value_units_per_bucket; + iter->specifics.linear.next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(h, value_units_per_bucket); + + iter->_next_fp = iter_linear_next; +} + +void hdr_iter_linear_set_value_units_per_bucket(struct hdr_iter* iter, int64_t value_units_per_bucket) +{ + iter->specifics.linear.value_units_per_bucket = value_units_per_bucket; +} + +/* ## ####### ###### ### ######## #### ######## ## ## ## ## #### ###### */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ### ### ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## #### #### ## ## */ +/* ## ## ## ## #### ## ## ######## ## ## ######### ## ### ## ## ## */ +/* ## ## ## ## ## ######### ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ######## ####### ###### ## ## ## ## #### ## ## ## ## ## #### ###### */ + +static bool log_iter_next(struct hdr_iter *iter) +{ + struct hdr_iter_log* logarithmic = &iter->specifics.log; + + logarithmic->count_added_in_this_iteration_step = 0; + + if (has_next(iter) || + next_value_greater_than_reporting_level_upper_bound( + iter, logarithmic->next_value_reporting_level_lowest_equivalent)) + { + do + { + if (iter->value >= logarithmic->next_value_reporting_level_lowest_equivalent) + { + update_iterated_values(iter, logarithmic->next_value_reporting_level); + + logarithmic->next_value_reporting_level *= (int64_t)logarithmic->log_base; + logarithmic->next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(iter->h, logarithmic->next_value_reporting_level); + + return true; + } + + if (!move_next(iter)) + { + return true; + } + + logarithmic->count_added_in_this_iteration_step += iter->count; + } + while (true); + } + + return false; +} + +void hdr_iter_log_init( + struct hdr_iter* iter, + const struct hdr_histogram* h, + int64_t value_units_first_bucket, + double log_base) +{ + hdr_iter_init(iter, h); + iter->specifics.log.count_added_in_this_iteration_step = 0; + iter->specifics.log.log_base = log_base; + iter->specifics.log.next_value_reporting_level = value_units_first_bucket; + iter->specifics.log.next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(h, value_units_first_bucket); + + iter->_next_fp = log_iter_next; +} + +/* Printing. */ + +static const char* format_head_string(format_type format) +{ + switch (format) + { + case CSV: + return "%s,%s,%s,%s\n"; + case CLASSIC: + default: + return "%12s %12s %12s %12s\n\n"; + } +} + +static const char CLASSIC_FOOTER[] = + "#[Mean = %12.3f, StdDeviation = %12.3f]\n" + "#[Max = %12.3f, Total count = %12" PRIu64 "]\n" + "#[Buckets = %12d, SubBuckets = %12d]\n"; + +int hdr_percentiles_print( + struct hdr_histogram* h, FILE* stream, int32_t ticks_per_half_distance, + double value_scale, format_type format) +{ + char line_format[25]; + const char* head_format; + int rc = 0; + struct hdr_iter iter; + struct hdr_iter_percentiles * percentiles; + + format_line_string(line_format, 25, h->significant_figures, format); + head_format = format_head_string(format); + + hdr_iter_percentile_init(&iter, h, ticks_per_half_distance); + + if (fprintf( + stream, head_format, + "Value", "Percentile", "TotalCount", "1/(1-Percentile)") < 0) + { + rc = EIO; + goto cleanup; + } + + percentiles = &iter.specifics.percentiles; + while (hdr_iter_next(&iter)) + { + double value = iter.highest_equivalent_value / value_scale; + double percentile = percentiles->percentile / 100.0; + int64_t total_count = iter.cumulative_count; + double inverted_percentile = (1.0 / (1.0 - percentile)); + + if (fprintf( + stream, line_format, value, percentile, total_count, inverted_percentile) < 0) + { + rc = EIO; + goto cleanup; + } + } + + if (CLASSIC == format) + { + double mean = hdr_mean(h) / value_scale; + double stddev = hdr_stddev(h) / value_scale; + double max = hdr_max(h) / value_scale; + + if (fprintf( + stream, CLASSIC_FOOTER, mean, stddev, max, + h->total_count, h->bucket_count, h->sub_bucket_count) < 0) + { + rc = EIO; + goto cleanup; + } + } + + cleanup: + return rc; +} diff --git a/deps/hdr_histogram/hdr_histogram.h b/deps/hdr_histogram/hdr_histogram.h new file mode 100644 index 000000000..c26c56b20 --- /dev/null +++ b/deps/hdr_histogram/hdr_histogram.h @@ -0,0 +1,509 @@ +/** + * hdr_histogram.h + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + * + * The source for the hdr_histogram utilises a few C99 constructs, specifically + * the use of stdint/stdbool and inline variable declaration. + */ + +#ifndef HDR_HISTOGRAM_H +#define HDR_HISTOGRAM_H 1 + +#include +#include +#include + +struct hdr_histogram +{ + int64_t lowest_trackable_value; + int64_t highest_trackable_value; + int32_t unit_magnitude; + int32_t significant_figures; + int32_t sub_bucket_half_count_magnitude; + int32_t sub_bucket_half_count; + int64_t sub_bucket_mask; + int32_t sub_bucket_count; + int32_t bucket_count; + int64_t min_value; + int64_t max_value; + int32_t normalizing_index_offset; + double conversion_ratio; + int32_t counts_len; + int64_t total_count; + int64_t* counts; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Allocate the memory and initialise the hdr_histogram. + * + * Due to the size of the histogram being the result of some reasonably + * involved math on the input parameters this function it is tricky to stack allocate. + * The histogram should be released with hdr_close + * + * @param lowest_trackable_value The smallest possible value to be put into the + * histogram. + * @param highest_trackable_value The largest possible value to be put into the + * histogram. + * @param significant_figures The level of precision for this histogram, i.e. the number + * of figures in a decimal number that will be maintained. E.g. a value of 3 will mean + * the results from the histogram will be accurate up to the first three digits. Must + * be a value between 1 and 5 (inclusive). + * @param result Output parameter to capture allocated histogram. + * @return 0 on success, EINVAL if lowest_trackable_value is < 1 or the + * significant_figure value is outside of the allowed range, ENOMEM if malloc + * failed. + */ +int hdr_init( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram** result); + +/** + * Free the memory and close the hdr_histogram. + * + * @param h The histogram you want to close. + */ +void hdr_close(struct hdr_histogram* h); + +/** + * Allocate the memory and initialise the hdr_histogram. This is the equivalent of calling + * hdr_init(1, highest_trackable_value, significant_figures, result); + * + * @deprecated use hdr_init. + */ +int hdr_alloc(int64_t highest_trackable_value, int significant_figures, struct hdr_histogram** result); + + +/** + * Reset a histogram to zero - empty out a histogram and re-initialise it + * + * If you want to re-use an existing histogram, but reset everything back to zero, this + * is the routine to use. + * + * @param h The histogram you want to reset to empty. + * + */ +void hdr_reset(struct hdr_histogram* h); + +/** + * Get the memory size of the hdr_histogram. + * + * @param h "This" pointer + * @return The amount of memory used by the hdr_histogram in bytes + */ +size_t hdr_get_memory_size(struct hdr_histogram* h); + +/** + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at construction time. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_value(struct hdr_histogram* h, int64_t value); + +/** + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at construction time. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_value_atomic(struct hdr_histogram* h, int64_t value); + +/** + * Records count values in the histogram, will round this value of to a + * precision at or better than the significant_figure specified at construction + * time. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @return false if any value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_values(struct hdr_histogram* h, int64_t value, int64_t count); + +/** + * Records count values in the histogram, will round this value of to a + * precision at or better than the significant_figure specified at construction + * time. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @return false if any value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count); + +/** + * Record a value in the histogram and backfill based on an expected interval. + * + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at contruction time. This is specifically used + * for recording latency. If the value is larger than the expected_interval then the + * latency recording system has experienced co-ordinated omission. This method fills in the + * values that would have occured had the client providing the load not been blocked. + + * @param h "This" pointer + * @param value Value to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t expexcted_interval); + +/** + * Record a value in the histogram and backfill based on an expected interval. + * + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at contruction time. This is specifically used + * for recording latency. If the value is larger than the expected_interval then the + * latency recording system has experienced co-ordinated omission. This method fills in the + * values that would have occured had the client providing the load not been blocked. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_value_atomic(struct hdr_histogram* h, int64_t value, int64_t expexcted_interval); + +/** + * Record a value in the histogram 'count' times. Applies the same correcting logic + * as 'hdr_record_corrected_value'. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_values(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval); + +/** + * Record a value in the histogram 'count' times. Applies the same correcting logic + * as 'hdr_record_corrected_value'. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval); + +/** + * Adds all of the values from 'from' to 'this' histogram. Will return the + * number of values that are dropped when copying. Values will be dropped + * if they around outside of h.lowest_trackable_value and + * h.highest_trackable_value. + * + * @param h "This" pointer + * @param from Histogram to copy values from. + * @return The number of values dropped when copying. + */ +int64_t hdr_add(struct hdr_histogram* h, const struct hdr_histogram* from); + +/** + * Adds all of the values from 'from' to 'this' histogram. Will return the + * number of values that are dropped when copying. Values will be dropped + * if they around outside of h.lowest_trackable_value and + * h.highest_trackable_value. + * + * @param h "This" pointer + * @param from Histogram to copy values from. + * @return The number of values dropped when copying. + */ +int64_t hdr_add_while_correcting_for_coordinated_omission( + struct hdr_histogram* h, struct hdr_histogram* from, int64_t expected_interval); + +/** + * Get minimum value from the histogram. Will return 2^63-1 if the histogram + * is empty. + * + * @param h "This" pointer + */ +int64_t hdr_min(const struct hdr_histogram* h); + +/** + * Get maximum value from the histogram. Will return 0 if the histogram + * is empty. + * + * @param h "This" pointer + */ +int64_t hdr_max(const struct hdr_histogram* h); + +/** + * Get the value at a specific percentile. + * + * @param h "This" pointer. + * @param percentile The percentile to get the value for + */ +int64_t hdr_value_at_percentile(const struct hdr_histogram* h, double percentile); + +/** + * Gets the standard deviation for the values in the histogram. + * + * @param h "This" pointer + * @return The standard deviation + */ +double hdr_stddev(const struct hdr_histogram* h); + +/** + * Gets the mean for the values in the histogram. + * + * @param h "This" pointer + * @return The mean + */ +double hdr_mean(const struct hdr_histogram* h); + +/** + * Determine if two values are equivalent with the histogram's resolution. + * Where "equivalent" means that value samples recorded for any two + * equivalent values are counted in a common total count. + * + * @param h "This" pointer + * @param a first value to compare + * @param b second value to compare + * @return 'true' if values are equivalent with the histogram's resolution. + */ +bool hdr_values_are_equivalent(const struct hdr_histogram* h, int64_t a, int64_t b); + +/** + * Get the lowest value that is equivalent to the given value within the histogram's resolution. + * Where "equivalent" means that value samples recorded for any two + * equivalent values are counted in a common total count. + * + * @param h "This" pointer + * @param value The given value + * @return The lowest value that is equivalent to the given value within the histogram's resolution. + */ +int64_t hdr_lowest_equivalent_value(const struct hdr_histogram* h, int64_t value); + +/** + * Get the count of recorded values at a specific value + * (to within the histogram resolution at the value level). + * + * @param h "This" pointer + * @param value The value for which to provide the recorded count + * @return The total count of values recorded in the histogram within the value range that is + * {@literal >=} lowestEquivalentValue(value) and {@literal <=} highestEquivalentValue(value) + */ +int64_t hdr_count_at_value(const struct hdr_histogram* h, int64_t value); + +int64_t hdr_count_at_index(const struct hdr_histogram* h, int32_t index); + +int64_t hdr_value_at_index(const struct hdr_histogram* h, int32_t index); + +struct hdr_iter_percentiles +{ + bool seen_last_value; + int32_t ticks_per_half_distance; + double percentile_to_iterate_to; + double percentile; +}; + +struct hdr_iter_recorded +{ + int64_t count_added_in_this_iteration_step; +}; + +struct hdr_iter_linear +{ + int64_t value_units_per_bucket; + int64_t count_added_in_this_iteration_step; + int64_t next_value_reporting_level; + int64_t next_value_reporting_level_lowest_equivalent; +}; + +struct hdr_iter_log +{ + double log_base; + int64_t count_added_in_this_iteration_step; + int64_t next_value_reporting_level; + int64_t next_value_reporting_level_lowest_equivalent; +}; + +/** + * The basic iterator. This is a generic structure + * that supports all of the types of iteration. Use + * the appropriate initialiser to get the desired + * iteration. + * + * @ + */ +struct hdr_iter +{ + const struct hdr_histogram* h; + /** raw index into the counts array */ + int32_t counts_index; + /** snapshot of the length at the time the iterator is created */ + int64_t total_count; + /** value directly from array for the current counts_index */ + int64_t count; + /** sum of all of the counts up to and including the count at this index */ + int64_t cumulative_count; + /** The current value based on counts_index */ + int64_t value; + int64_t highest_equivalent_value; + int64_t lowest_equivalent_value; + int64_t median_equivalent_value; + int64_t value_iterated_from; + int64_t value_iterated_to; + + union + { + struct hdr_iter_percentiles percentiles; + struct hdr_iter_recorded recorded; + struct hdr_iter_linear linear; + struct hdr_iter_log log; + } specifics; + + bool (* _next_fp)(struct hdr_iter* iter); + +}; + +/** + * Initalises the basic iterator. + * + * @param itr 'This' pointer + * @param h The histogram to iterate over + */ +void hdr_iter_init(struct hdr_iter* iter, const struct hdr_histogram* h); + +/** + * Initialise the iterator for use with percentiles. + */ +void hdr_iter_percentile_init(struct hdr_iter* iter, const struct hdr_histogram* h, int32_t ticks_per_half_distance); + +/** + * Initialise the iterator for use with recorded values. + */ +void hdr_iter_recorded_init(struct hdr_iter* iter, const struct hdr_histogram* h); + +/** + * Initialise the iterator for use with linear values. + */ +void hdr_iter_linear_init( + struct hdr_iter* iter, + const struct hdr_histogram* h, + int64_t value_units_per_bucket); + +/** + * Update the iterator value units per bucket + */ +void hdr_iter_linear_set_value_units_per_bucket(struct hdr_iter* iter, int64_t value_units_per_bucket); + +/** + * Initialise the iterator for use with logarithmic values + */ +void hdr_iter_log_init( + struct hdr_iter* iter, + const struct hdr_histogram* h, + int64_t value_units_first_bucket, + double log_base); + +/** + * Iterate to the next value for the iterator. If there are no more values + * available return faluse. + * + * @param itr 'This' pointer + * @return 'false' if there are no values remaining for this iterator. + */ +bool hdr_iter_next(struct hdr_iter* iter); + +typedef enum +{ + CLASSIC, + CSV +} format_type; + +/** + * Print out a percentile based histogram to the supplied stream. Note that + * this call will not flush the FILE, this is left up to the user. + * + * @param h 'This' pointer + * @param stream The FILE to write the output to + * @param ticks_per_half_distance The number of iteration steps per half-distance to 100% + * @param value_scale Scale the output values by this amount + * @param format_type Format to use, e.g. CSV. + * @return 0 on success, error code on failure. EIO if an error occurs writing + * the output. + */ +int hdr_percentiles_print( + struct hdr_histogram* h, FILE* stream, int32_t ticks_per_half_distance, + double value_scale, format_type format); + +/** +* Internal allocation methods, used by hdr_dbl_histogram. +*/ +struct hdr_histogram_bucket_config +{ + int64_t lowest_trackable_value; + int64_t highest_trackable_value; + int64_t unit_magnitude; + int64_t significant_figures; + int32_t sub_bucket_half_count_magnitude; + int32_t sub_bucket_half_count; + int64_t sub_bucket_mask; + int32_t sub_bucket_count; + int32_t bucket_count; + int32_t counts_len; +}; + +int hdr_calculate_bucket_config( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram_bucket_config* cfg); + +void hdr_init_preallocated(struct hdr_histogram* h, struct hdr_histogram_bucket_config* cfg); + +int64_t hdr_size_of_equivalent_value_range(const struct hdr_histogram* h, int64_t value); + +int64_t hdr_next_non_equivalent_value(const struct hdr_histogram* h, int64_t value); + +int64_t hdr_median_equivalent_value(const struct hdr_histogram* h, int64_t value); + +/** + * Used to reset counters after importing data manuallying into the histogram, used by the logging code + * and other custom serialisation tools. + */ +void hdr_reset_internal_counters(struct hdr_histogram* h); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/deps/hiredis/.gitignore b/deps/hiredis/.gitignore index 8e50b5434..056959ffe 100644 --- a/deps/hiredis/.gitignore +++ b/deps/hiredis/.gitignore @@ -6,3 +6,4 @@ /*.a /*.pc *.dSYM +tags diff --git a/deps/hiredis/.travis.yml b/deps/hiredis/.travis.yml index dd8e0e73d..f9a9460ff 100644 --- a/deps/hiredis/.travis.yml +++ b/deps/hiredis/.travis.yml @@ -1,5 +1,4 @@ language: c -sudo: false compiler: - gcc - clang @@ -8,17 +7,34 @@ os: - linux - osx +dist: bionic + branches: only: - staging - trying - master + - /^release\/.*$/ + +install: + - if [ "$BITS" == "64" ]; then + wget https://github.com/redis/redis/archive/6.0.6.tar.gz; + tar -xzvf 6.0.6.tar.gz; + pushd redis-6.0.6 && BUILD_TLS=yes make && export PATH=$PWD/src:$PATH && popd; + fi before_script: - - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then brew update; brew install redis; fi + - if [ "$TRAVIS_OS_NAME" == "osx" ]; then + curl -O https://distfiles.macports.org/MacPorts/MacPorts-2.6.2-10.13-HighSierra.pkg; + sudo installer -pkg MacPorts-2.6.2-10.13-HighSierra.pkg -target /; + export PATH=$PATH:/opt/local/bin && sudo port -v selfupdate; + sudo port -N install openssl redis; + fi; addons: apt: + sources: + - sourceline: 'ppa:chris-lea/redis-server' packages: - libc6-dbg - libc6-dev @@ -27,14 +43,20 @@ addons: - libc6-dbg:i386 - gcc-multilib - g++-multilib + - libssl-dev + - libssl-dev:i386 - valgrind + - redis env: - BITS="32" - BITS="64" script: - - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DHIREDIS_SSL:BOOL=ON"; + - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON"; + if [ "$BITS" == "64" ]; then + EXTRA_CMAKE_OPTS="$EXTRA_CMAKE_OPTS -DENABLE_SSL_TESTS:BOOL=ON"; + fi; if [ "$TRAVIS_OS_NAME" == "osx" ]; then if [ "$BITS" == "32" ]; then CFLAGS="-m32 -Werror"; @@ -58,12 +80,24 @@ script: fi; fi; export CFLAGS CXXFLAGS LDFLAGS TEST_PREFIX EXTRA_CMAKE_OPTS + - make && make clean; + if [ "$TRAVIS_OS_NAME" == "osx" ]; then + if [ "$BITS" == "64" ]; then + OPENSSL_PREFIX="$(ls -d /usr/local/Cellar/openssl@1.1/*)" USE_SSL=1 make; + fi; + else + USE_SSL=1 make; + fi; - mkdir build/ && cd build/ - cmake .. ${EXTRA_CMAKE_OPTS} - make VERBOSE=1 - - ctest -V + - if [ "$BITS" == "64" ]; then + TEST_SSL=1 SKIPS_AS_FAILS=1 ctest -V; + else + SKIPS_AS_FAILS=1 ctest -V; + fi; -matrix: +jobs: include: # Windows MinGW cross compile on Linux - os: linux @@ -89,9 +123,9 @@ matrix: - eval "${MATRIX_EVAL}" install: - choco install ninja + - choco install -y memurai-developer script: - mkdir build && cd build - - cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 && - cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release && - ninja -v' - - ctest -V + - cmd.exe //C 'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat' amd64 '&&' + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DENABLE_EXAMPLES=ON '&&' ninja -v + - ./hiredis-test.exe diff --git a/deps/hiredis/CHANGELOG.md b/deps/hiredis/CHANGELOG.md index d1d37e515..271f1fcf3 100644 --- a/deps/hiredis/CHANGELOG.md +++ b/deps/hiredis/CHANGELOG.md @@ -1,28 +1,175 @@ -### 1.0.0 (unreleased) +## [1.0.0](https://github.com/redis/hiredis/tree/v1.0.0) - (2020-08-03) + +Announcing Hiredis v1.0.0, which adds support for RESP3, SSL connections, allocator injection, and better Windows support! :tada: + +_A big thanks to everyone who helped with this release. The following list includes everyone who contributed at least five lines, sorted by lines contributed._ :sparkling_heart: + +[Michael Grunder](https://github.com/michael-grunder), [Yossi Gottlieb](https://github.com/yossigo), +[Mark Nunberg](https://github.com/mnunberg), [Marcus Geelnard](https://github.com/mbitsnbites), +[Justin Brewer](https://github.com/justinbrewer), [Valentino Geron](https://github.com/valentinogeron), +[Minun Dragonation](https://github.com/dragonation), [Omri Steiner](https://github.com/OmriSteiner), +[Sangmoon Yi](https://github.com/jman-krafton), [Jinjiazh](https://github.com/jinjiazhang), +[Odin Hultgren Van Der Horst](https://github.com/Miniwoffer), [Muhammad Zahalqa](https://github.com/tryfinally), +[Nick Rivera](https://github.com/heronr), [Qi Yang](https://github.com/movebean), +[kevin1018](https://github.com/kevin1018) + +[Full Changelog](https://github.com/redis/hiredis/compare/v0.14.1...v1.0.0) **BREAKING CHANGES**: -* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now - protocol errors. This is consistent with the RESP specification. On 32-bit - platforms, the upper bound is lowered to `SIZE_MAX`. +* `redisOptions` now has two timeout fields. One for connecting, and one for commands. If you're presently using `options->timeout` you will need to change it to use `options->connect_timeout`. (See [example](https://github.com/redis/hiredis/commit/38b5ae543f5c99eb4ccabbe277770fc6bc81226f#diff-86ba39d37aa829c8c82624cce4f049fbL36)) -* Change `redisReply.len` to `size_t`, as it denotes the the size of a string - - User code should compare this to `size_t` values as well. If it was used to - compare to other values, casting might be necessary or can be removed, if - casting was applied before. - -### 0.x.x (unreleased) -**BREAKING CHANGES**: - -* Change `redisReply.len` to `size_t`, as it denotes the the size of a string - -User code should compare this to `size_t` values as well. -If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before. +* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now protocol errors. This is consistent + with the RESP specification. On 32-bit platforms, the upper bound is lowered to `SIZE_MAX`. * `redisReplyObjectFunctions.createArray` now takes `size_t` for its length parameter. +**New features:** +- Support for RESP3 + [\#697](https://github.com/redis/hiredis/pull/697), + [\#805](https://github.com/redis/hiredis/pull/805), + [\#819](https://github.com/redis/hiredis/pull/819), + [\#841](https://github.com/redis/hiredis/pull/841) + ([Yossi Gottlieb](https://github.com/yossigo), [Michael Grunder](https://github.com/michael-grunder)) +- Support for SSL connections + [\#645](https://github.com/redis/hiredis/pull/645), + [\#699](https://github.com/redis/hiredis/pull/699), + [\#702](https://github.com/redis/hiredis/pull/702), + [\#708](https://github.com/redis/hiredis/pull/708), + [\#711](https://github.com/redis/hiredis/pull/711), + [\#821](https://github.com/redis/hiredis/pull/821), + [more](https://github.com/redis/hiredis/pulls?q=is%3Apr+is%3Amerged+SSL) + ([Mark Nunberg](https://github.com/mnunberg), [Yossi Gottlieb](https://github.com/yossigo)) +- Run-time allocator injection + [\#800](https://github.com/redis/hiredis/pull/800) + ([Michael Grunder](https://github.com/michael-grunder)) +- Improved Windows support (including MinGW and Windows CI) + [\#652](https://github.com/redis/hiredis/pull/652), + [\#663](https://github.com/redis/hiredis/pull/663) + ([Marcus Geelnard](https://www.bitsnbites.eu/author/m/)) +- Adds support for distinct connect and command timeouts + [\#839](https://github.com/redis/hiredis/pull/839), + [\#829](https://github.com/redis/hiredis/pull/829) + ([Valentino Geron](https://github.com/valentinogeron)) +- Add generic pointer and destructor to `redisContext` that users can use for context. + [\#855](https://github.com/redis/hiredis/pull/855) + ([Michael Grunder](https://github.com/michael-grunder)) + +**Closed issues (that involved code changes):** + +- Makefile does not install TLS libraries [\#809](https://github.com/redis/hiredis/issues/809) +- redisConnectWithOptions should not set command timeout [\#722](https://github.com/redis/hiredis/issues/722), [\#829](https://github.com/redis/hiredis/pull/829) ([valentinogeron](https://github.com/valentinogeron)) +- Fix integer overflow in `sdsrange` [\#827](https://github.com/redis/hiredis/issues/827) +- INFO & CLUSTER commands failed when using RESP3 [\#802](https://github.com/redis/hiredis/issues/802) +- Windows compatibility patches [\#687](https://github.com/redis/hiredis/issues/687), [\#838](https://github.com/redis/hiredis/issues/838), [\#842](https://github.com/redis/hiredis/issues/842) +- RESP3 PUSH messages incorrectly use pending callback [\#825](https://github.com/redis/hiredis/issues/825) +- Asynchronous PSUBSCRIBE command fails when using RESP3 [\#815](https://github.com/redis/hiredis/issues/815) +- New SSL API [\#804](https://github.com/redis/hiredis/issues/804), [\#813](https://github.com/redis/hiredis/issues/813) +- Hard-coded limit of nested reply depth [\#794](https://github.com/redis/hiredis/issues/794) +- Fix TCP_NODELAY in Windows/OSX [\#679](https://github.com/redis/hiredis/issues/679), [\#690](https://github.com/redis/hiredis/issues/690), [\#779](https://github.com/redis/hiredis/issues/779), [\#785](https://github.com/redis/hiredis/issues/785), +- Added timers to libev adapter. [\#778](https://github.com/redis/hiredis/issues/778), [\#795](https://github.com/redis/hiredis/pull/795) +- Initialization discards const qualifier [\#777](https://github.com/redis/hiredis/issues/777) +- \[BUG\]\[MinGW64\] Error setting socket timeout [\#775](https://github.com/redis/hiredis/issues/775) +- undefined reference to hi_malloc [\#769](https://github.com/redis/hiredis/issues/769) +- hiredis pkg-config file incorrectly ignores multiarch libdir spec'n [\#767](https://github.com/redis/hiredis/issues/767) +- Don't use -G to build shared object on Solaris [\#757](https://github.com/redis/hiredis/issues/757) +- error when make USE\_SSL=1 [\#748](https://github.com/redis/hiredis/issues/748) +- Allow to change SSL Mode [\#646](https://github.com/redis/hiredis/issues/646) +- hiredis/adapters/libevent.h memleak [\#618](https://github.com/redis/hiredis/issues/618) +- redisLibuvPoll crash when server closes the connetion [\#545](https://github.com/redis/hiredis/issues/545) +- about redisAsyncDisconnect question [\#518](https://github.com/redis/hiredis/issues/518) +- hiredis adapters libuv error for help [\#508](https://github.com/redis/hiredis/issues/508) +- API/ABI changes analysis [\#506](https://github.com/redis/hiredis/issues/506) +- Memory leak patch in Redis [\#502](https://github.com/redis/hiredis/issues/502) +- Remove the depth limitation [\#421](https://github.com/redis/hiredis/issues/421) + +**Merged pull requests:** + +- Move SSL management to a distinct private pointer [\#855](https://github.com/redis/hiredis/pull/855) ([michael-grunder](https://github.com/michael-grunder)) +- Move include to sockcompat.h to maintain style [\#850](https://github.com/redis/hiredis/pull/850) ([michael-grunder](https://github.com/michael-grunder)) +- Remove erroneous tag and add license to push example [\#849](https://github.com/redis/hiredis/pull/849) ([michael-grunder](https://github.com/michael-grunder)) +- fix windows compiling with mingw [\#848](https://github.com/redis/hiredis/pull/848) ([rmalizia44](https://github.com/rmalizia44)) +- Some Windows quality of life improvements. [\#846](https://github.com/redis/hiredis/pull/846) ([michael-grunder](https://github.com/michael-grunder)) +- Use \_WIN32 define instead of WIN32 [\#845](https://github.com/redis/hiredis/pull/845) ([michael-grunder](https://github.com/michael-grunder)) +- Non Linux CI fixes [\#844](https://github.com/redis/hiredis/pull/844) ([michael-grunder](https://github.com/michael-grunder)) +- Resp3 oob push support [\#841](https://github.com/redis/hiredis/pull/841) ([michael-grunder](https://github.com/michael-grunder)) +- fix \#785: defer TCP\_NODELAY in async tcp connections [\#836](https://github.com/redis/hiredis/pull/836) ([OmriSteiner](https://github.com/OmriSteiner)) +- sdsrange overflow fix [\#830](https://github.com/redis/hiredis/pull/830) ([michael-grunder](https://github.com/michael-grunder)) +- Use explicit pointer casting for c++ compatibility [\#826](https://github.com/redis/hiredis/pull/826) ([aureus1](https://github.com/aureus1)) +- Document allocator injection and completeness fix in test.c [\#824](https://github.com/redis/hiredis/pull/824) ([michael-grunder](https://github.com/michael-grunder)) +- Use unique names for allocator struct members [\#823](https://github.com/redis/hiredis/pull/823) ([michael-grunder](https://github.com/michael-grunder)) +- New SSL API to replace redisSecureConnection\(\). [\#821](https://github.com/redis/hiredis/pull/821) ([yossigo](https://github.com/yossigo)) +- Add logic to handle RESP3 push messages [\#819](https://github.com/redis/hiredis/pull/819) ([michael-grunder](https://github.com/michael-grunder)) +- Use standrad isxdigit instead of custom helper function. [\#814](https://github.com/redis/hiredis/pull/814) ([tryfinally](https://github.com/tryfinally)) +- Fix missing SSL build/install options. [\#812](https://github.com/redis/hiredis/pull/812) ([yossigo](https://github.com/yossigo)) +- Add link to ABI tracker [\#808](https://github.com/redis/hiredis/pull/808) ([michael-grunder](https://github.com/michael-grunder)) +- Resp3 verbatim string support [\#805](https://github.com/redis/hiredis/pull/805) ([michael-grunder](https://github.com/michael-grunder)) +- Allow users to replace allocator and handle OOM everywhere. [\#800](https://github.com/redis/hiredis/pull/800) ([michael-grunder](https://github.com/michael-grunder)) +- Remove nested depth limitation. [\#797](https://github.com/redis/hiredis/pull/797) ([michael-grunder](https://github.com/michael-grunder)) +- Attempt to fix compilation on Solaris [\#796](https://github.com/redis/hiredis/pull/796) ([michael-grunder](https://github.com/michael-grunder)) +- Support timeouts in libev adapater [\#795](https://github.com/redis/hiredis/pull/795) ([michael-grunder](https://github.com/michael-grunder)) +- Fix pkgconfig when installing to a custom lib dir [\#793](https://github.com/redis/hiredis/pull/793) ([michael-grunder](https://github.com/michael-grunder)) +- Fix USE\_SSL=1 make/cmake on OSX and CMake tests [\#789](https://github.com/redis/hiredis/pull/789) ([michael-grunder](https://github.com/michael-grunder)) +- Use correct libuv call on Windows [\#784](https://github.com/redis/hiredis/pull/784) ([michael-grunder](https://github.com/michael-grunder)) +- Added CMake package config and fixed hiredis\_ssl on Windows [\#783](https://github.com/redis/hiredis/pull/783) ([michael-grunder](https://github.com/michael-grunder)) +- CMake: Set hiredis\_ssl shared object version. [\#780](https://github.com/redis/hiredis/pull/780) ([yossigo](https://github.com/yossigo)) +- Win32 tests and timeout fix [\#776](https://github.com/redis/hiredis/pull/776) ([michael-grunder](https://github.com/michael-grunder)) +- Provides an optional cleanup callback for async data. [\#768](https://github.com/redis/hiredis/pull/768) ([heronr](https://github.com/heronr)) +- Housekeeping fixes [\#764](https://github.com/redis/hiredis/pull/764) ([michael-grunder](https://github.com/michael-grunder)) +- install alloc.h [\#756](https://github.com/redis/hiredis/pull/756) ([ch1aki](https://github.com/ch1aki)) +- fix spelling mistakes [\#746](https://github.com/redis/hiredis/pull/746) ([ShooterIT](https://github.com/ShooterIT)) +- Free the reply in redisGetReply when passed NULL [\#741](https://github.com/redis/hiredis/pull/741) ([michael-grunder](https://github.com/michael-grunder)) +- Fix dead code in sslLogCallback relating to should\_log variable. [\#737](https://github.com/redis/hiredis/pull/737) ([natoscott](https://github.com/natoscott)) +- Fix typo in dict.c. [\#731](https://github.com/redis/hiredis/pull/731) ([Kevin-Xi](https://github.com/Kevin-Xi)) +- Adding an option to DISABLE\_TESTS [\#727](https://github.com/redis/hiredis/pull/727) ([pbotros](https://github.com/pbotros)) +- Update README with SSL support. [\#720](https://github.com/redis/hiredis/pull/720) ([yossigo](https://github.com/yossigo)) +- Fixes leaks in unit tests [\#715](https://github.com/redis/hiredis/pull/715) ([michael-grunder](https://github.com/michael-grunder)) +- SSL Tests [\#711](https://github.com/redis/hiredis/pull/711) ([yossigo](https://github.com/yossigo)) +- SSL Reorganization [\#708](https://github.com/redis/hiredis/pull/708) ([yossigo](https://github.com/yossigo)) +- Fix MSVC build. [\#706](https://github.com/redis/hiredis/pull/706) ([yossigo](https://github.com/yossigo)) +- SSL: Properly report SSL\_connect\(\) errors. [\#702](https://github.com/redis/hiredis/pull/702) ([yossigo](https://github.com/yossigo)) +- Silent SSL trace to stdout by default. [\#699](https://github.com/redis/hiredis/pull/699) ([yossigo](https://github.com/yossigo)) +- Port RESP3 support from Redis. [\#697](https://github.com/redis/hiredis/pull/697) ([yossigo](https://github.com/yossigo)) +- Removed whitespace before newline [\#691](https://github.com/redis/hiredis/pull/691) ([Miniwoffer](https://github.com/Miniwoffer)) +- Add install adapters header files [\#688](https://github.com/redis/hiredis/pull/688) ([kevin1018](https://github.com/kevin1018)) +- Remove unnecessary null check before free [\#684](https://github.com/redis/hiredis/pull/684) ([qlyoung](https://github.com/qlyoung)) +- redisReaderGetReply leak memory [\#671](https://github.com/redis/hiredis/pull/671) ([movebean](https://github.com/movebean)) +- fix timeout code in windows [\#670](https://github.com/redis/hiredis/pull/670) ([jman-krafton](https://github.com/jman-krafton)) +- test: fix errstr matching for musl libc [\#665](https://github.com/redis/hiredis/pull/665) ([ghost](https://github.com/ghost)) +- Windows: MinGW fixes and Windows Travis builders [\#663](https://github.com/redis/hiredis/pull/663) ([mbitsnbites](https://github.com/mbitsnbites)) +- The setsockopt and getsockopt API diffs from BSD socket and WSA one [\#662](https://github.com/redis/hiredis/pull/662) ([dragonation](https://github.com/dragonation)) +- Fix Compile Error On Windows \(Visual Studio\) [\#658](https://github.com/redis/hiredis/pull/658) ([jinjiazhang](https://github.com/jinjiazhang)) +- Fix NXDOMAIN test case [\#653](https://github.com/redis/hiredis/pull/653) ([michael-grunder](https://github.com/michael-grunder)) +- Add MinGW support [\#652](https://github.com/redis/hiredis/pull/652) ([mbitsnbites](https://github.com/mbitsnbites)) +- SSL Support [\#645](https://github.com/redis/hiredis/pull/645) ([mnunberg](https://github.com/mnunberg)) +- Fix Invalid argument after redisAsyncConnectUnix [\#644](https://github.com/redis/hiredis/pull/644) ([codehz](https://github.com/codehz)) +- Makefile: use predefined AR [\#632](https://github.com/redis/hiredis/pull/632) ([Mic92](https://github.com/Mic92)) +- FreeBSD build fix [\#628](https://github.com/redis/hiredis/pull/628) ([devnexen](https://github.com/devnexen)) +- Fix errors not propagating properly with libuv.h. [\#624](https://github.com/redis/hiredis/pull/624) ([yossigo](https://github.com/yossigo)) +- Update README.md [\#621](https://github.com/redis/hiredis/pull/621) ([Crunsher](https://github.com/Crunsher)) +- Fix redisBufferRead documentation [\#620](https://github.com/redis/hiredis/pull/620) ([hacst](https://github.com/hacst)) +- Add CPPFLAGS to REAL\_CFLAGS [\#614](https://github.com/redis/hiredis/pull/614) ([thomaslee](https://github.com/thomaslee)) +- Update createArray to take size\_t [\#597](https://github.com/redis/hiredis/pull/597) ([justinbrewer](https://github.com/justinbrewer)) +- fix common realloc mistake and add null check more [\#580](https://github.com/redis/hiredis/pull/580) ([charsyam](https://github.com/charsyam)) +- Proper error reporting for connect failures [\#578](https://github.com/redis/hiredis/pull/578) ([mnunberg](https://github.com/mnunberg)) + +\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* + +## [1.0.0-rc1](https://github.com/redis/hiredis/tree/v1.0.0-rc1) - (2020-07-29) + +_Note: There were no changes to code between v1.0.0-rc1 and v1.0.0 so see v1.0.0 for changelog_ + +### 0.14.1 (2020-03-13) + +* Adds safe allocation wrappers (CVE-2020-7105, #747, #752) (Michael Grunder) + ### 0.14.0 (2018-09-25) +**BREAKING CHANGES**: + +* Change `redisReply.len` to `size_t`, as it denotes the the size of a string + + User code should compare this to `size_t` values as well. + If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before. * Make string2ll static to fix conflict with Redis (Tom Lee [c3188b]) * Use -dynamiclib instead of -shared for OSX (Ryan Schmidt [a65537]) @@ -196,4 +343,3 @@ The parser, standalone since v0.12.0, can now be compiled on Windows ### 0.10.0 * See commit log. - diff --git a/deps/hiredis/CMakeLists.txt b/deps/hiredis/CMakeLists.txt index 9e78894f3..f86c9b70b 100644 --- a/deps/hiredis/CMakeLists.txt +++ b/deps/hiredis/CMakeLists.txt @@ -3,6 +3,8 @@ INCLUDE(GNUInstallDirs) PROJECT(hiredis) OPTION(ENABLE_SSL "Build hiredis_ssl for SSL support" OFF) +OPTION(DISABLE_TESTS "If tests should be compiled or not" OFF) +OPTION(ENABLE_SSL_TESTS, "Should we test SSL connections" OFF) MACRO(getVersionBit name) SET(VERSION_REGEX "^#define ${name} (.+)$") @@ -22,7 +24,8 @@ PROJECT(hiredis VERSION "${VERSION}") SET(ENABLE_EXAMPLES OFF CACHE BOOL "Enable building hiredis examples") -ADD_LIBRARY(hiredis SHARED +SET(hiredis_sources + alloc.c async.c dict.c hiredis.c @@ -31,20 +34,32 @@ ADD_LIBRARY(hiredis SHARED sds.c sockcompat.c) +SET(hiredis_sources ${hiredis_sources}) + +IF(WIN32) + ADD_COMPILE_DEFINITIONS(_CRT_SECURE_NO_WARNINGS WIN32_LEAN_AND_MEAN) +ENDIF() + +ADD_LIBRARY(hiredis SHARED ${hiredis_sources}) + SET_TARGET_PROPERTIES(hiredis - PROPERTIES + PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE VERSION "${HIREDIS_SONAME}") IF(WIN32 OR MINGW) TARGET_LINK_LIBRARIES(hiredis PRIVATE ws2_32) ENDIF() -TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC .) + +TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC $ $) CONFIGURE_FILE(hiredis.pc.in hiredis.pc @ONLY) INSTALL(TARGETS hiredis - DESTINATION "${CMAKE_INSTALL_LIBDIR}") + EXPORT hiredis-targets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) -INSTALL(FILES hiredis.h read.h sds.h async.h +INSTALL(FILES hiredis.h read.h sds.h async.h alloc.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) INSTALL(DIRECTORY adapters @@ -53,6 +68,26 @@ INSTALL(DIRECTORY adapters INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +export(EXPORT hiredis-targets + FILE "${CMAKE_CURRENT_BINARY_DIR}/hiredis-targets.cmake" + NAMESPACE hiredis::) + +SET(CMAKE_CONF_INSTALL_DIR share/hiredis) +SET(INCLUDE_INSTALL_DIR include) +include(CMakePackageConfigHelpers) +configure_package_config_file(hiredis-config.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/hiredis-config.cmake + INSTALL_DESTINATION ${CMAKE_CONF_INSTALL_DIR} + PATH_VARS INCLUDE_INSTALL_DIR) + +INSTALL(EXPORT hiredis-targets + FILE hiredis-targets.cmake + NAMESPACE hiredis:: + DESTINATION ${CMAKE_CONF_INSTALL_DIR}) + +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis-config.cmake + DESTINATION ${CMAKE_CONF_INSTALL_DIR}) + + IF(ENABLE_SSL) IF (NOT OPENSSL_ROOT_DIR) IF (APPLE) @@ -60,26 +95,66 @@ IF(ENABLE_SSL) ENDIF() ENDIF() FIND_PACKAGE(OpenSSL REQUIRED) - ADD_LIBRARY(hiredis_ssl SHARED + SET(hiredis_ssl_sources ssl.c) + ADD_LIBRARY(hiredis_ssl SHARED + ${hiredis_ssl_sources}) + + IF (APPLE) + SET_PROPERTY(TARGET hiredis_ssl PROPERTY LINK_FLAGS "-Wl,-undefined -Wl,dynamic_lookup") + ENDIF() + + SET_TARGET_PROPERTIES(hiredis_ssl + PROPERTIES + WINDOWS_EXPORT_ALL_SYMBOLS TRUE + VERSION "${HIREDIS_SONAME}") + TARGET_INCLUDE_DIRECTORIES(hiredis_ssl PRIVATE "${OPENSSL_INCLUDE_DIR}") TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE ${OPENSSL_LIBRARIES}) + IF (WIN32 OR MINGW) + TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE hiredis) + ENDIF() CONFIGURE_FILE(hiredis_ssl.pc.in hiredis_ssl.pc @ONLY) INSTALL(TARGETS hiredis_ssl - DESTINATION "${CMAKE_INSTALL_LIBDIR}") + EXPORT hiredis_ssl-targets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) INSTALL(FILES hiredis_ssl.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + + export(EXPORT hiredis_ssl-targets + FILE "${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-targets.cmake" + NAMESPACE hiredis::) + + SET(CMAKE_CONF_INSTALL_DIR share/hiredis_ssl) + configure_package_config_file(hiredis_ssl-config.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-config.cmake + INSTALL_DESTINATION ${CMAKE_CONF_INSTALL_DIR} + PATH_VARS INCLUDE_INSTALL_DIR) + + INSTALL(EXPORT hiredis_ssl-targets + FILE hiredis_ssl-targets.cmake + NAMESPACE hiredis:: + DESTINATION ${CMAKE_CONF_INSTALL_DIR}) + + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-config.cmake + DESTINATION ${CMAKE_CONF_INSTALL_DIR}) ENDIF() -IF(NOT (WIN32 OR MINGW)) +IF(NOT DISABLE_TESTS) ENABLE_TESTING() ADD_EXECUTABLE(hiredis-test test.c) - TARGET_LINK_LIBRARIES(hiredis-test hiredis) + IF(ENABLE_SSL_TESTS) + ADD_DEFINITIONS(-DHIREDIS_TEST_SSL=1) + TARGET_LINK_LIBRARIES(hiredis-test hiredis hiredis_ssl) + ELSE() + TARGET_LINK_LIBRARIES(hiredis-test hiredis) + ENDIF() ADD_TEST(NAME hiredis-test COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test.sh) ENDIF() diff --git a/deps/hiredis/Makefile b/deps/hiredis/Makefile index 841990d5c..a9e87dd82 100644 --- a/deps/hiredis/Makefile +++ b/deps/hiredis/Makefile @@ -3,16 +3,16 @@ # Copyright (C) 2010-2011 Pieter Noordhuis # This file is released under the BSD license, see the COPYING file -OBJ=net.o hiredis.o sds.o async.o read.o sockcompat.o +OBJ=alloc.o net.o hiredis.o sds.o async.o read.o sockcompat.o SSL_OBJ=ssl.o -EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib +EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib hiredis-example-push ifeq ($(USE_SSL),1) EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl endif TESTS=hiredis-test LIBNAME=libhiredis -SSL_LIBNAME=libhiredis_ssl PKGCONFNAME=hiredis.pc +SSL_LIBNAME=libhiredis_ssl SSL_PKGCONFNAME=hiredis_ssl.pc HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}') @@ -55,12 +55,17 @@ STLIBSUFFIX=a DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX) -SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX) + DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) STLIBNAME=$(LIBNAME).$(STLIBSUFFIX) -SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX) STLIB_MAKE_CMD=$(AR) rcs +SSL_DYLIB_MINOR_NAME=$(SSL_LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) +SSL_DYLIB_MAJOR_NAME=$(SSL_LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) +SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX) +SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX) +SSL_DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(SSL_DYLIB_MINOR_NAME) + # Platform-specific overrides uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') @@ -80,13 +85,22 @@ else endif ifeq ($(uname_S),SunOS) + IS_SUN_CC=$(shell sh -c '$(CC) -V 2>&1 |egrep -i -c "sun|studio"') + ifeq ($(IS_SUN_CC),1) + SUN_SHARED_FLAG=-G + else + SUN_SHARED_FLAG=-shared + endif REAL_LDFLAGS+= -ldl -lnsl -lsocket - DYLIB_MAKE_CMD=$(CC) -G -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS) + DYLIB_MAKE_CMD=$(CC) $(SUN_SHARED_FLAG) -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS) + SSL_DYLIB_MAKE_CMD=$(CC) $(SUN_SHARED_FLAG) -o $(SSL_DYLIBNAME) -h $(SSL_DYLIB_MINOR_NAME) $(LDFLAGS) $(SSL_LDFLAGS) endif ifeq ($(uname_S),Darwin) DYLIBSUFFIX=dylib DYLIB_MINOR_NAME=$(LIBNAME).$(HIREDIS_SONAME).$(DYLIBSUFFIX) DYLIB_MAKE_CMD=$(CC) -dynamiclib -Wl,-install_name,$(PREFIX)/$(LIBRARY_PATH)/$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) + SSL_DYLIB_MAKE_CMD=$(CC) -dynamiclib -Wl,-install_name,$(PREFIX)/$(LIBRARY_PATH)/$(SSL_DYLIB_MINOR_NAME) -o $(SSL_DYLIBNAME) $(LDFLAGS) $(SSL_LDFLAGS) + DYLIB_PLUGIN=-Wl,-undefined -Wl,dynamic_lookup endif all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME) @@ -95,15 +109,16 @@ all: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) endif # Deps (use make dep to generate this) -async.o: async.c fmacros.h async.h hiredis.h read.h sds.h net.h dict.c dict.h -dict.o: dict.c fmacros.h dict.h -hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h win32.h -net.o: net.c fmacros.h net.h hiredis.h read.h sds.h sockcompat.h win32.h -read.o: read.c fmacros.h read.h sds.h -sds.o: sds.c sds.h +alloc.o: alloc.c fmacros.h alloc.h +async.o: async.c fmacros.h alloc.h async.h hiredis.h read.h sds.h net.h dict.c dict.h win32.h async_private.h +dict.o: dict.c fmacros.h alloc.h dict.h +hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h alloc.h net.h async.h win32.h +net.o: net.c fmacros.h net.h hiredis.h read.h sds.h alloc.h sockcompat.h win32.h +read.o: read.c fmacros.h alloc.h read.h sds.h win32.h +sds.o: sds.c sds.h sdsalloc.h alloc.h sockcompat.o: sockcompat.c sockcompat.h -ssl.o: ssl.c hiredis.h -test.o: test.c fmacros.h hiredis.h read.h sds.h +ssl.o: ssl.c hiredis.h read.h sds.h alloc.h async.h win32.h async_private.h +test.o: test.c fmacros.h hiredis.h read.h sds.h alloc.h net.h sockcompat.h win32.h $(DYLIBNAME): $(OBJ) $(DYLIB_MAKE_CMD) -o $(DYLIBNAME) $(OBJ) $(REAL_LDFLAGS) @@ -112,7 +127,7 @@ $(STLIBNAME): $(OBJ) $(STLIB_MAKE_CMD) $(STLIBNAME) $(OBJ) $(SSL_DYLIBNAME): $(SSL_OBJ) - $(DYLIB_MAKE_CMD) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(SSL_LDFLAGS) + $(SSL_DYLIB_MAKE_CMD) $(DYLIB_PLUGIN) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(LDFLAGS) $(SSL_LDFLAGS) $(SSL_STLIBNAME): $(SSL_OBJ) $(STLIB_MAKE_CMD) $(SSL_STLIBNAME) $(SSL_OBJ) @@ -146,6 +161,7 @@ hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME) hiredis-example-ssl: examples/example-ssl.c $(STLIBNAME) $(SSL_STLIBNAME) $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) + ifndef AE_DIR hiredis-example-ae: @echo "Please specify AE_DIR (e.g. /src)" @@ -180,13 +196,19 @@ endif hiredis-example: examples/example.c $(STLIBNAME) $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS) +hiredis-example-push: examples/example-push.c $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS) + examples: $(EXAMPLES) TEST_LIBS = $(STLIBNAME) ifeq ($(USE_SSL),1) - TEST_LIBS += $(SSL_STLIBNAME) -lssl -lcrypto -lpthread + TEST_LIBS += $(SSL_STLIBNAME) + TEST_LDFLAGS = $(SSL_LDFLAGS) -lssl -lcrypto -lpthread endif + hiredis-test: test.o $(TEST_LIBS) + $(CC) -o $@ $(REAL_CFLAGS) -I. $^ $(REAL_LDFLAGS) $(TEST_LDFLAGS) hiredis-%: %.o $(STLIBNAME) $(CC) $(REAL_CFLAGS) -o $@ $< $(TEST_LIBS) $(REAL_LDFLAGS) @@ -221,7 +243,7 @@ $(PKGCONFNAME): hiredis.h @echo Libs: -L\$${libdir} -lhiredis >> $@ @echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@ -$(SSL_PKGCONFNAME): hiredis.h +$(SSL_PKGCONFNAME): hiredis_ssl.h @echo "Generating $@ for pkgconfig..." @echo prefix=$(PREFIX) > $@ @echo exec_prefix=\$${prefix} >> $@ @@ -237,7 +259,7 @@ $(SSL_PKGCONFNAME): hiredis.h install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL_LIBRARY_PATH) - $(INSTALL) hiredis.h async.h read.h sds.h $(INSTALL_INCLUDE_PATH) + $(INSTALL) hiredis.h async.h read.h sds.h alloc.h $(INSTALL_INCLUDE_PATH) $(INSTALL) adapters/*.h $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL) $(DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(DYLIB_MINOR_NAME) cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MINOR_NAME) $(DYLIBNAME) @@ -245,6 +267,19 @@ install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) mkdir -p $(INSTALL_PKGCONF_PATH) $(INSTALL) $(PKGCONFNAME) $(INSTALL_PKGCONF_PATH) +ifeq ($(USE_SSL),1) +install: install-ssl + +install-ssl: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) + mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_LIBRARY_PATH) + $(INSTALL) hiredis_ssl.h $(INSTALL_INCLUDE_PATH) + $(INSTALL) $(SSL_DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(SSL_DYLIB_MINOR_NAME) + cd $(INSTALL_LIBRARY_PATH) && ln -sf $(SSL_DYLIB_MINOR_NAME) $(SSL_DYLIBNAME) + $(INSTALL) $(SSL_STLIBNAME) $(INSTALL_LIBRARY_PATH) + mkdir -p $(INSTALL_PKGCONF_PATH) + $(INSTALL) $(SSL_PKGCONFNAME) $(INSTALL_PKGCONF_PATH) +endif + 32bit: @echo "" @echo "WARNING: if this fails under Linux you probably need to install libc6-dev-i386" diff --git a/deps/hiredis/README.md b/deps/hiredis/README.md index c0b432f07..3a22553ea 100644 --- a/deps/hiredis/README.md +++ b/deps/hiredis/README.md @@ -1,6 +1,6 @@ [![Build Status](https://travis-ci.org/redis/hiredis.png)](https://travis-ci.org/redis/hiredis) -**This Readme reflects the latest changed in the master branch. See [v0.13.3](https://github.com/redis/hiredis/tree/v0.13.3) for the Readme and documentation for the latest release.** +**This Readme reflects the latest changed in the master branch. See [v1.0.0](https://github.com/redis/hiredis/tree/v1.0.0) for the Readme and documentation for the latest release ([API/ABI history](https://abi-laboratory.pro/?view=timeline&l=hiredis)).** # HIREDIS @@ -24,12 +24,31 @@ The library comes with multiple APIs. There is the ## Upgrading to `1.0.0` -Version 1.0.0 marks a stable release of hiredis. +Version 1.0.0 marks the first stable release of Hiredis. It includes some minor breaking changes, mostly to make the exposed API more uniform and self-explanatory. It also bundles the updated `sds` library, to sync up with upstream and Redis. -For most applications a recompile against the new hiredis should be enough. For code changes see the [Changelog](CHANGELOG.md). +_Note: As described below, a few member names have been changed but most applications should be able to upgrade with minor code changes and recompiling._ + +## IMPORTANT: Breaking changes from `0.14.1` -> `1.0.0` + +* `redisContext` has two additional members (`free_privdata`, and `privctx`). +* `redisOptions.timeout` has been renamed to `redisOptions.connect_timeout`, and we've added `redisOptions.command_timeout`. +* `redisReplyObjectFunctions.createArray` now takes `size_t` instead of `int` for its length parameter. + +## IMPORTANT: Breaking changes when upgrading from 0.13.x -> 0.14.x + +Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now +protocol errors. This is consistent with the RESP specification. On 32-bit +platforms, the upper bound is lowered to `SIZE_MAX`. + +Change `redisReply.len` to `size_t`, as it denotes the the size of a string + +User code should compare this to `size_t` values as well. If it was used to +compare to other values, casting might be necessary or can be removed, if +casting was applied before. + ## Upgrading from `<0.9.0` Version 0.9.0 is a major overhaul of hiredis in every aspect. However, upgrading existing @@ -110,6 +129,8 @@ The standard replies that `redisCommand` are of the type `redisReply`. The `type` field in the `redisReply` should be used to test what kind of reply was received: +### RESP2 + * **`REDIS_REPLY_STATUS`**: * The command replied with a status reply. The status string can be accessed using `reply->str`. The length of this string can be accessed using `reply->len`. @@ -134,16 +155,51 @@ was received: and can be accessed via `reply->element[..index..]`. Redis may reply with nested arrays but this is fully supported. +### RESP3 + +Hiredis also supports every new `RESP3` data type which are as follows. For more information about the protocol see the `RESP3` [specification.](https://github.com/antirez/RESP3/blob/master/spec.md) + +* **`REDIS_REPLY_DOUBLE`**: + * The command replied with a double-precision floating point number. + The value is stored as a string in the `str` member, and can be converted with `strtod` or similar. + +* **`REDIS_REPLY_BOOL`**: + * A boolean true/false reply. + The value is stored in the `integer` member and will be either `0` or `1`. + +* **`REDIS_REPLY_MAP`**: + * An array with the added invariant that there will always be an even number of elements. + The MAP is functionally equivelant to `REDIS_REPLY_ARRAY` except for the previously mentioned invariant. + +* **`REDIS_REPLY_SET`**: + * An array response where each entry is unique. + Like the MAP type, the data is identical to an array response except there are no duplicate values. + +* **`REDIS_REPLY_PUSH`**: + * An array that can be generated spontaneously by Redis. + This array response will always contain at least two subelements. The first contains the type of `PUSH` message (e.g. `message`, or `invalidate`), and the second being a sub-array with the `PUSH` payload itself. + +* **`REDIS_REPLY_ATTR`**: + * An array structurally identical to a `MAP` but intended as meta-data about a reply. + _As of Redis 6.0.6 this reply type is not used in Redis_ + +* **`REDIS_REPLY_BIGNUM`**: + * A string representing an arbitrarily large signed or unsigned integer value. + The number will be encoded as a string in the `str` member of `redisReply`. + +* **`REDIS_REPLY_VERB`**: + * A verbatim string, intended to be presented to the user without modification. + The string payload is stored in the `str` memeber, and type data is stored in the `vtype` member (e.g. `txt` for raw text or `md` for markdown). + Replies should be freed using the `freeReplyObject()` function. Note that this function will take care of freeing sub-reply objects contained in arrays and nested arrays, so there is no need for the user to free the sub replies (it is actually harmful and will corrupt the memory). -**Important:** the current version of hiredis (0.10.0) frees replies when the +**Important:** the current version of hiredis (1.0.0) frees replies when the asynchronous API is used. This means you should not call `freeReplyObject` when you use this API. The reply is cleaned up by hiredis _after_ the callback -returns. This behavior will probably change in future releases, so make sure to -keep an eye on the changelog when upgrading (see issue #39). +returns. We may introduce a flag to make this configurable in future versions of the library. ### Cleaning up @@ -205,16 +261,16 @@ a single call to `read(2)`): redisReply *reply; redisAppendCommand(context,"SET foo bar"); redisAppendCommand(context,"GET foo"); -redisGetReply(context,&reply); // reply for SET +redisGetReply(context,(void *)&reply); // reply for SET freeReplyObject(reply); -redisGetReply(context,&reply); // reply for GET +redisGetReply(context,(void *)&reply); // reply for GET freeReplyObject(reply); ``` This API can also be used to implement a blocking subscriber: ```c reply = redisCommand(context,"SUBSCRIBE foo"); freeReplyObject(reply); -while(redisGetReply(context,&reply) == REDIS_OK) { +while(redisGetReply(context,(void *)&reply) == REDIS_OK) { // consume message freeReplyObject(reply); } @@ -404,9 +460,199 @@ This should be done only in order to maximize performances when working with large payloads. The context should be set back to `REDIS_READER_MAX_BUF` again as soon as possible in order to prevent allocation of useless memory. +### Reader max array elements + +By default the hiredis reply parser sets the maximum number of multi-bulk elements +to 2^32 - 1 or 4,294,967,295 entries. If you need to process multi-bulk replies +with more than this many elements you can set the value higher or to zero, meaning +unlimited with: +```c +context->reader->maxelements = 0; +``` + +## SSL/TLS Support + +### Building + +SSL/TLS support is not built by default and requires an explicit flag: + + make USE_SSL=1 + +This requires OpenSSL development package (e.g. including header files to be +available. + +When enabled, SSL/TLS support is built into extra `libhiredis_ssl.a` and +`libhiredis_ssl.so` static/dynamic libraries. This leaves the original libraries +unaffected so no additional dependencies are introduced. + +### Using it + +First, you'll need to make sure you include the SSL header file: + +```c +#include "hiredis.h" +#include "hiredis_ssl.h" +``` + +You will also need to link against `libhiredis_ssl`, **in addition** to +`libhiredis` and add `-lssl -lcrypto` to satisfy its dependencies. + +Hiredis implements SSL/TLS on top of its normal `redisContext` or +`redisAsyncContext`, so you will need to establish a connection first and then +initiate an SSL/TLS handshake. + +#### Hiredis OpenSSL Wrappers + +Before Hiredis can negotiate an SSL/TLS connection, it is necessary to +initialize OpenSSL and create a context. You can do that in two ways: + +1. Work directly with the OpenSSL API to initialize the library's global context + and create `SSL_CTX *` and `SSL *` contexts. With an `SSL *` object you can + call `redisInitiateSSL()`. +2. Work with a set of Hiredis-provided wrappers around OpenSSL, create a + `redisSSLContext` object to hold configuration and use + `redisInitiateSSLWithContext()` to initiate the SSL/TLS handshake. + +```c +/* An Hiredis SSL context. It holds SSL configuration and can be reused across + * many contexts. + */ +redisSSLContext *ssl; + +/* An error variable to indicate what went wrong, if the context fails to + * initialize. + */ +redisSSLContextError ssl_error; + +/* Initialize global OpenSSL state. + * + * You should call this only once when your app initializes, and only if + * you don't explicitly or implicitly initialize OpenSSL it elsewhere. + */ +redisInitOpenSSL(); + +/* Create SSL context */ +ssl = redisCreateSSLContext( + "cacertbundle.crt", /* File name of trusted CA/ca bundle file, optional */ + "/path/to/certs", /* Path of trusted certificates, optional */ + "client_cert.pem", /* File name of client certificate file, optional */ + "client_key.pem", /* File name of client private key, optional */ + "redis.mydomain.com", /* Server name to request (SNI), optional */ + &ssl_error + ) != REDIS_OK) { + printf("SSL error: %s\n", redisSSLContextGetError(ssl_error); + /* Abort... */ + } + +/* Create Redis context and establish connection */ +c = redisConnect("localhost", 6443); +if (c == NULL || c->err) { + /* Handle error and abort... */ +} + +/* Negotiate SSL/TLS */ +if (redisInitiateSSLWithContext(c, ssl) != REDIS_OK) { + /* Handle error, in c->err / c->errstr */ +} +``` + +## RESP3 PUSH replies +Redis 6.0 introduced PUSH replies with the reply-type `>`. These messages are generated spontaneously and can arrive at any time, so must be handled using callbacks. + +### Default behavior +Hiredis installs handlers on `redisContext` and `redisAsyncContext` by default, which will intercept and free any PUSH replies detected. This means existing code will work as-is after upgrading to Redis 6 and switching to `RESP3`. + +### Custom PUSH handler prototypes +The callback prototypes differ between `redisContext` and `redisAsyncContext`. + +#### redisContext +```c +void my_push_handler(void *privdata, void *reply) { + /* Handle the reply */ + + /* Note: We need to free the reply in our custom handler for + blocking contexts. This lets us keep the reply if + we want. */ + freeReplyObject(reply); +} +``` + +#### redisAsyncContext +```c +void my_async_push_handler(redisAsyncContext *ac, void *reply) { + /* Handle the reply */ + + /* Note: Because async hiredis always frees replies, you should + not call freeReplyObject in an async push callback. */ +} +``` + +### Installing a custom handler +There are two ways to set your own PUSH handlers. + +1. Set `push_cb` or `async_push_cb` in the `redisOptions` struct and connect with `redisConnectWithOptions` or `redisAsyncConnectWithOptions`. + ```c + redisOptions = {0}; + REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); + options->push_cb = my_push_handler; + redisContext *context = redisConnectWithOptions(&options); + ``` +2. Call `redisSetPushCallback` or `redisAsyncSetPushCallback` on a connected context. + ```c + redisContext *context = redisConnect("127.0.0.1", 6379); + redisSetPushCallback(context, my_push_handler); + ``` + + _Note `redisSetPushCallback` and `redisAsyncSetPushCallback` both return any currently configured handler, making it easy to override and then return to the old value._ + +### Specifying no handler +If you have a unique use-case where you don't want hiredis to automatically intercept and free PUSH replies, you will want to configure no handler at all. This can be done in two ways. +1. Set the `REDIS_OPT_NO_PUSH_AUTOFREE` flag in `redisOptions` and leave the callback function pointer `NULL`. + ```c + redisOptions = {0}; + REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); + options->options |= REDIS_OPT_NO_PUSH_AUTOFREE; + redisContext *context = redisConnectWithOptions(&options); + ``` +3. Call `redisSetPushCallback` with `NULL` once connected. + ```c + redisContext *context = redisConnect("127.0.0.1", 6379); + redisSetPushCallback(context, NULL); + ``` + + _Note: With no handler configured, calls to `redisCommand` may generate more than one reply, so this strategy is only applicable when there's some kind of blocking`redisGetReply()` loop (e.g. `MONITOR` or `SUBSCRIBE` workloads)._ + +## Allocator injection + +Hiredis uses a pass-thru structure of function pointers defined in [alloc.h](https://github.com/redis/hiredis/blob/f5d25850/alloc.h#L41) that contain the currently configured allocation and deallocation functions. By default they just point to libc (`malloc`, `calloc`, `realloc`, etc). + +### Overriding + +One can override the allocators like so: + +```c +hiredisAllocFuncs myfuncs = { + .mallocFn = my_malloc, + .callocFn = my_calloc, + .reallocFn = my_realloc, + .strdupFn = my_strdup, + .freeFn = my_free, +}; + +// Override allocators (function returns current allocators if needed) +hiredisAllocFuncs orig = hiredisSetAllocators(&myfuncs); +``` + +To reset the allocators to their default libc function simply call: + +```c +hiredisResetAllocators(); +``` + ## AUTHORS -Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and -Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. -Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and -Jan-Erik Rediger (janerik at fnordig dot com) +Salvatore Sanfilippo (antirez at gmail),\ +Pieter Noordhuis (pcnoordhuis at gmail)\ +Michael Grunder (michael dot grunder at gmail) + +_Hiredis is released under the BSD license._ diff --git a/deps/hiredis/adapters/ae.h b/deps/hiredis/adapters/ae.h index 5c551c2ed..660d82eb0 100644 --- a/deps/hiredis/adapters/ae.h +++ b/deps/hiredis/adapters/ae.h @@ -96,7 +96,7 @@ static void redisAeCleanup(void *privdata) { redisAeEvents *e = (redisAeEvents*)privdata; redisAeDelRead(privdata); redisAeDelWrite(privdata); - free(e); + hi_free(e); } static int redisAeAttach(aeEventLoop *loop, redisAsyncContext *ac) { @@ -108,7 +108,10 @@ static int redisAeAttach(aeEventLoop *loop, redisAsyncContext *ac) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisAeEvents*)malloc(sizeof(*e)); + e = (redisAeEvents*)hi_malloc(sizeof(*e)); + if (e == NULL) + return REDIS_ERR; + e->context = ac; e->loop = loop; e->fd = c->fd; diff --git a/deps/hiredis/adapters/glib.h b/deps/hiredis/adapters/glib.h index e0a6411d3..ad59dd142 100644 --- a/deps/hiredis/adapters/glib.h +++ b/deps/hiredis/adapters/glib.h @@ -134,6 +134,9 @@ redis_source_new (redisAsyncContext *ac) g_return_val_if_fail(ac != NULL, NULL); source = (RedisSource *)g_source_new(&source_funcs, sizeof *source); + if (source == NULL) + return NULL; + source->ac = ac; source->poll_fd.fd = c->fd; source->poll_fd.events = 0; diff --git a/deps/hiredis/adapters/ivykis.h b/deps/hiredis/adapters/ivykis.h index 6a12a868a..179f6ab52 100644 --- a/deps/hiredis/adapters/ivykis.h +++ b/deps/hiredis/adapters/ivykis.h @@ -43,7 +43,7 @@ static void redisIvykisCleanup(void *privdata) { redisIvykisEvents *e = (redisIvykisEvents*)privdata; iv_fd_unregister(&e->fd); - free(e); + hi_free(e); } static int redisIvykisAttach(redisAsyncContext *ac) { @@ -55,7 +55,10 @@ static int redisIvykisAttach(redisAsyncContext *ac) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisIvykisEvents*)malloc(sizeof(*e)); + e = (redisIvykisEvents*)hi_malloc(sizeof(*e)); + if (e == NULL) + return REDIS_ERR; + e->context = ac; /* Register functions to start/stop listening for events */ diff --git a/deps/hiredis/adapters/libev.h b/deps/hiredis/adapters/libev.h index 2bf8d521f..e1e7bbd99 100644 --- a/deps/hiredis/adapters/libev.h +++ b/deps/hiredis/adapters/libev.h @@ -41,6 +41,7 @@ typedef struct redisLibevEvents { struct ev_loop *loop; int reading, writing; ev_io rev, wev; + ev_timer timer; } redisLibevEvents; static void redisLibevReadEvent(EV_P_ ev_io *watcher, int revents) { @@ -103,11 +104,39 @@ static void redisLibevDelWrite(void *privdata) { } } +static void redisLibevStopTimer(void *privdata) { + redisLibevEvents *e = (redisLibevEvents*)privdata; + struct ev_loop *loop = e->loop; + ((void)loop); + ev_timer_stop(EV_A_ &e->timer); +} + static void redisLibevCleanup(void *privdata) { redisLibevEvents *e = (redisLibevEvents*)privdata; redisLibevDelRead(privdata); redisLibevDelWrite(privdata); - free(e); + redisLibevStopTimer(privdata); + hi_free(e); +} + +static void redisLibevTimeout(EV_P_ ev_timer *timer, int revents) { + ((void)revents); + redisLibevEvents *e = (redisLibevEvents*)timer->data; + redisAsyncHandleTimeout(e->context); +} + +static void redisLibevSetTimeout(void *privdata, struct timeval tv) { + redisLibevEvents *e = (redisLibevEvents*)privdata; + struct ev_loop *loop = e->loop; + ((void)loop); + + if (!ev_is_active(&e->timer)) { + ev_init(&e->timer, redisLibevTimeout); + e->timer.data = e; + } + + e->timer.repeat = tv.tv_sec + tv.tv_usec / 1000000.00; + ev_timer_again(EV_A_ &e->timer); } static int redisLibevAttach(EV_P_ redisAsyncContext *ac) { @@ -119,14 +148,16 @@ static int redisLibevAttach(EV_P_ redisAsyncContext *ac) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisLibevEvents*)malloc(sizeof(*e)); + e = (redisLibevEvents*)hi_calloc(1, sizeof(*e)); + if (e == NULL) + return REDIS_ERR; + e->context = ac; #if EV_MULTIPLICITY e->loop = loop; #else e->loop = NULL; #endif - e->reading = e->writing = 0; e->rev.data = e; e->wev.data = e; @@ -136,6 +167,7 @@ static int redisLibevAttach(EV_P_ redisAsyncContext *ac) { ac->ev.addWrite = redisLibevAddWrite; ac->ev.delWrite = redisLibevDelWrite; ac->ev.cleanup = redisLibevCleanup; + ac->ev.scheduleTimer = redisLibevSetTimeout; ac->ev.data = e; /* Initialize read/write events */ diff --git a/deps/hiredis/adapters/libevent.h b/deps/hiredis/adapters/libevent.h index a4952776c..9150979bc 100644 --- a/deps/hiredis/adapters/libevent.h +++ b/deps/hiredis/adapters/libevent.h @@ -47,7 +47,7 @@ typedef struct redisLibeventEvents { } redisLibeventEvents; static void redisLibeventDestroy(redisLibeventEvents *e) { - free(e); + hi_free(e); } static void redisLibeventHandler(int fd, short event, void *arg) { @@ -152,7 +152,10 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisLibeventEvents*)calloc(1, sizeof(*e)); + e = (redisLibeventEvents*)hi_calloc(1, sizeof(*e)); + if (e == NULL) + return REDIS_ERR; + e->context = ac; /* Register functions to start/stop listening for events */ diff --git a/deps/hiredis/adapters/libuv.h b/deps/hiredis/adapters/libuv.h index 39ef7cf5e..c120b1b39 100644 --- a/deps/hiredis/adapters/libuv.h +++ b/deps/hiredis/adapters/libuv.h @@ -73,7 +73,7 @@ static void redisLibuvDelWrite(void *privdata) { static void on_close(uv_handle_t* handle) { redisLibuvEvents* p = (redisLibuvEvents*)handle->data; - free(p); + hi_free(p); } @@ -98,15 +98,13 @@ static int redisLibuvAttach(redisAsyncContext* ac, uv_loop_t* loop) { ac->ev.delWrite = redisLibuvDelWrite; ac->ev.cleanup = redisLibuvCleanup; - redisLibuvEvents* p = (redisLibuvEvents*)malloc(sizeof(*p)); - - if (!p) { - return REDIS_ERR; - } + redisLibuvEvents* p = (redisLibuvEvents*)hi_malloc(sizeof(*p)); + if (p == NULL) + return REDIS_ERR; memset(p, 0, sizeof(*p)); - if (uv_poll_init(loop, &p->handle, c->fd) != 0) { + if (uv_poll_init_socket(loop, &p->handle, c->fd) != 0) { return REDIS_ERR; } diff --git a/deps/hiredis/adapters/macosx.h b/deps/hiredis/adapters/macosx.h index 72121f606..3c87f1b2f 100644 --- a/deps/hiredis/adapters/macosx.h +++ b/deps/hiredis/adapters/macosx.h @@ -27,7 +27,7 @@ static int freeRedisRunLoop(RedisRunLoop* redisRunLoop) { CFSocketInvalidate(redisRunLoop->socketRef); CFRelease(redisRunLoop->socketRef); } - free(redisRunLoop); + hi_free(redisRunLoop); } return REDIS_ERR; } @@ -80,8 +80,9 @@ static int redisMacOSAttach(redisAsyncContext *redisAsyncCtx, CFRunLoopRef runLo /* Nothing should be attached when something is already attached */ if( redisAsyncCtx->ev.data != NULL ) return REDIS_ERR; - RedisRunLoop* redisRunLoop = (RedisRunLoop*) calloc(1, sizeof(RedisRunLoop)); - if( !redisRunLoop ) return REDIS_ERR; + RedisRunLoop* redisRunLoop = (RedisRunLoop*) hi_calloc(1, sizeof(RedisRunLoop)); + if (redisRunLoop == NULL) + return REDIS_ERR; /* Setup redis stuff */ redisRunLoop->context = redisAsyncCtx; diff --git a/deps/hiredis/alloc.c b/deps/hiredis/alloc.c new file mode 100644 index 000000000..7fb6b35e7 --- /dev/null +++ b/deps/hiredis/alloc.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2020, Michael Grunder + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "fmacros.h" +#include "alloc.h" +#include +#include + +hiredisAllocFuncs hiredisAllocFns = { + .mallocFn = malloc, + .callocFn = calloc, + .reallocFn = realloc, + .strdupFn = strdup, + .freeFn = free, +}; + +/* Override hiredis' allocators with ones supplied by the user */ +hiredisAllocFuncs hiredisSetAllocators(hiredisAllocFuncs *override) { + hiredisAllocFuncs orig = hiredisAllocFns; + + hiredisAllocFns = *override; + + return orig; +} + +/* Reset allocators to use libc defaults */ +void hiredisResetAllocators(void) { + hiredisAllocFns = (hiredisAllocFuncs) { + .mallocFn = malloc, + .callocFn = calloc, + .reallocFn = realloc, + .strdupFn = strdup, + .freeFn = free, + }; +} + +#ifdef _WIN32 + +void *hi_malloc(size_t size) { + return hiredisAllocFns.mallocFn(size); +} + +void *hi_calloc(size_t nmemb, size_t size) { + return hiredisAllocFns.callocFn(nmemb, size); +} + +void *hi_realloc(void *ptr, size_t size) { + return hiredisAllocFns.reallocFn(ptr, size); +} + +char *hi_strdup(const char *str) { + return hiredisAllocFns.strdupFn(str); +} + +void hi_free(void *ptr) { + hiredisAllocFns.freeFn(ptr); +} + +#endif diff --git a/deps/hiredis/alloc.h b/deps/hiredis/alloc.h new file mode 100644 index 000000000..34a05f49f --- /dev/null +++ b/deps/hiredis/alloc.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2020, Michael Grunder + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef HIREDIS_ALLOC_H +#define HIREDIS_ALLOC_H + +#include /* for size_t */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Structure pointing to our actually configured allocators */ +typedef struct hiredisAllocFuncs { + void *(*mallocFn)(size_t); + void *(*callocFn)(size_t,size_t); + void *(*reallocFn)(void*,size_t); + char *(*strdupFn)(const char*); + void (*freeFn)(void*); +} hiredisAllocFuncs; + +hiredisAllocFuncs hiredisSetAllocators(hiredisAllocFuncs *ha); +void hiredisResetAllocators(void); + +#ifndef _WIN32 + +/* Hiredis' configured allocator function pointer struct */ +extern hiredisAllocFuncs hiredisAllocFns; + +static inline void *hi_malloc(size_t size) { + return hiredisAllocFns.mallocFn(size); +} + +static inline void *hi_calloc(size_t nmemb, size_t size) { + return hiredisAllocFns.callocFn(nmemb, size); +} + +static inline void *hi_realloc(void *ptr, size_t size) { + return hiredisAllocFns.reallocFn(ptr, size); +} + +static inline char *hi_strdup(const char *str) { + return hiredisAllocFns.strdupFn(str); +} + +static inline void hi_free(void *ptr) { + hiredisAllocFns.freeFn(ptr); +} + +#else + +void *hi_malloc(size_t size); +void *hi_calloc(size_t nmemb, size_t size); +void *hi_realloc(void *ptr, size_t size); +char *hi_strdup(const char *str); +void hi_free(void *ptr); + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* HIREDIS_ALLOC_H */ diff --git a/deps/hiredis/async.c b/deps/hiredis/async.c index 4f422d566..64ab601c9 100644 --- a/deps/hiredis/async.c +++ b/deps/hiredis/async.c @@ -30,6 +30,7 @@ */ #include "fmacros.h" +#include "alloc.h" #include #include #ifndef _MSC_VER @@ -46,18 +47,24 @@ #include "async_private.h" -/* Forward declaration of function in hiredis.c */ +/* Forward declarations of hiredis.c functions */ int __redisAppendCommand(redisContext *c, const char *cmd, size_t len); +void __redisSetError(redisContext *c, int type, const char *str); /* Functions managing dictionary of callbacks for pub/sub. */ static unsigned int callbackHash(const void *key) { return dictGenHashFunction((const unsigned char *)key, - sdslen((const sds)key)); + hi_sdslen((const hisds)key)); } static void *callbackValDup(void *privdata, const void *src) { ((void) privdata); - redisCallback *dup = malloc(sizeof(*dup)); + redisCallback *dup; + + dup = hi_malloc(sizeof(*dup)); + if (dup == NULL) + return NULL; + memcpy(dup,src,sizeof(*dup)); return dup; } @@ -66,20 +73,20 @@ static int callbackKeyCompare(void *privdata, const void *key1, const void *key2 int l1, l2; ((void) privdata); - l1 = sdslen((const sds)key1); - l2 = sdslen((const sds)key2); + l1 = hi_sdslen((const hisds)key1); + l2 = hi_sdslen((const hisds)key2); if (l1 != l2) return 0; return memcmp(key1,key2,l1) == 0; } static void callbackKeyDestructor(void *privdata, void *key) { ((void) privdata); - sdsfree((sds)key); + hi_sdsfree((hisds)key); } static void callbackValDestructor(void *privdata, void *val) { ((void) privdata); - free(val); + hi_free(val); } static dictType callbackDict = { @@ -93,10 +100,19 @@ static dictType callbackDict = { static redisAsyncContext *redisAsyncInitialize(redisContext *c) { redisAsyncContext *ac; + dict *channels = NULL, *patterns = NULL; - ac = realloc(c,sizeof(redisAsyncContext)); + channels = dictCreate(&callbackDict,NULL); + if (channels == NULL) + goto oom; + + patterns = dictCreate(&callbackDict,NULL); + if (patterns == NULL) + goto oom; + + ac = hi_realloc(c,sizeof(redisAsyncContext)); if (ac == NULL) - return NULL; + goto oom; c = &(ac->c); @@ -108,6 +124,7 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { ac->err = 0; ac->errstr = NULL; ac->data = NULL; + ac->dataCleanup = NULL; ac->ev.data = NULL; ac->ev.addRead = NULL; @@ -124,9 +141,14 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { ac->replies.tail = NULL; ac->sub.invalid.head = NULL; ac->sub.invalid.tail = NULL; - ac->sub.channels = dictCreate(&callbackDict,NULL); - ac->sub.patterns = dictCreate(&callbackDict,NULL); + ac->sub.channels = channels; + ac->sub.patterns = patterns; + return ac; +oom: + if (channels) dictRelease(channels); + if (patterns) dictRelease(patterns); + return NULL; } /* We want the error field to be accessible directly instead of requiring @@ -145,16 +167,26 @@ redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options) { redisContext *c; redisAsyncContext *ac; + /* Clear any erroneously set sync callback and flag that we don't want to + * use freeReplyObject by default. */ + myOptions.push_cb = NULL; + myOptions.options |= REDIS_OPT_NO_PUSH_AUTOFREE; + myOptions.options |= REDIS_OPT_NONBLOCK; c = redisConnectWithOptions(&myOptions); if (c == NULL) { return NULL; } + ac = redisAsyncInitialize(c); if (ac == NULL) { redisFree(c); return NULL; } + + /* Set any configured async push handler */ + redisAsyncSetPushCallback(ac, myOptions.async_push_cb); + __redisAsyncCopyError(ac); return ac; } @@ -214,7 +246,7 @@ static int __redisPushCallback(redisCallbackList *list, redisCallback *source) { redisCallback *cb; /* Copy callback from stack to heap */ - cb = malloc(sizeof(*cb)); + cb = hi_malloc(sizeof(*cb)); if (cb == NULL) return REDIS_ERR_OOM; @@ -242,7 +274,7 @@ static int __redisShiftCallback(redisCallbackList *list, redisCallback *target) /* Copy callback from heap to stack */ if (target != NULL) memcpy(target,cb,sizeof(*cb)); - free(cb); + hi_free(cb); return REDIS_OK; } return REDIS_ERR; @@ -257,6 +289,14 @@ static void __redisRunCallback(redisAsyncContext *ac, redisCallback *cb, redisRe } } +static void __redisRunPushCallback(redisAsyncContext *ac, redisReply *reply) { + if (ac->push_cb != NULL) { + ac->c.flags |= REDIS_IN_CALLBACK; + ac->push_cb(ac, reply); + ac->c.flags &= ~REDIS_IN_CALLBACK; + } +} + /* Helper function to free the context. */ static void __redisAsyncFree(redisAsyncContext *ac) { redisContext *c = &(ac->c); @@ -272,18 +312,28 @@ static void __redisAsyncFree(redisAsyncContext *ac) { while (__redisShiftCallback(&ac->sub.invalid,&cb) == REDIS_OK) __redisRunCallback(ac,&cb,NULL); - /* Run subscription callbacks callbacks with NULL reply */ - it = dictGetIterator(ac->sub.channels); - while ((de = dictNext(it)) != NULL) - __redisRunCallback(ac,dictGetEntryVal(de),NULL); - dictReleaseIterator(it); - dictRelease(ac->sub.channels); + /* Run subscription callbacks with NULL reply */ + if (ac->sub.channels) { + it = dictGetIterator(ac->sub.channels); + if (it != NULL) { + while ((de = dictNext(it)) != NULL) + __redisRunCallback(ac,dictGetEntryVal(de),NULL); + dictReleaseIterator(it); + } - it = dictGetIterator(ac->sub.patterns); - while ((de = dictNext(it)) != NULL) - __redisRunCallback(ac,dictGetEntryVal(de),NULL); - dictReleaseIterator(it); - dictRelease(ac->sub.patterns); + dictRelease(ac->sub.channels); + } + + if (ac->sub.patterns) { + it = dictGetIterator(ac->sub.patterns); + if (it != NULL) { + while ((de = dictNext(it)) != NULL) + __redisRunCallback(ac,dictGetEntryVal(de),NULL); + dictReleaseIterator(it); + } + + dictRelease(ac->sub.patterns); + } /* Signal event lib to clean up */ _EL_CLEANUP(ac); @@ -298,6 +348,10 @@ static void __redisAsyncFree(redisAsyncContext *ac) { } } + if (ac->dataCleanup) { + ac->dataCleanup(ac->data); + } + /* Cleanup self */ redisFree(c); } @@ -364,11 +418,11 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, dictEntry *de; int pvariant; char *stype; - sds sname; + hisds sname; /* Custom reply functions are not supported for pub/sub. This will fail * very hard when they are used... */ - if (reply->type == REDIS_REPLY_ARRAY) { + if (reply->type == REDIS_REPLY_ARRAY || reply->type == REDIS_REPLY_PUSH) { assert(reply->elements >= 2); assert(reply->element[0]->type == REDIS_REPLY_STRING); stype = reply->element[0]->str; @@ -381,7 +435,10 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, /* Locate the right callback */ assert(reply->element[1]->type == REDIS_REPLY_STRING); - sname = sdsnewlen(reply->element[1]->str,reply->element[1]->len); + sname = hi_sdsnewlen(reply->element[1]->str,reply->element[1]->len); + if (sname == NULL) + goto oom; + de = dictFind(callbacks,sname); if (de != NULL) { cb = dictGetEntryVal(de); @@ -409,12 +466,39 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, c->flags &= ~REDIS_SUBSCRIBED; } } - sdsfree(sname); + hi_sdsfree(sname); } else { /* Shift callback for invalid commands. */ __redisShiftCallback(&ac->sub.invalid,dstcb); } return REDIS_OK; +oom: + __redisSetError(&(ac->c), REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; +} + +#define redisIsSpontaneousPushReply(r) \ + (redisIsPushReply(r) && !redisIsSubscribeReply(r)) + +static int redisIsSubscribeReply(redisReply *reply) { + char *str; + size_t len, off; + + /* We will always have at least one string with the subscribe/message type */ + if (reply->elements < 1 || reply->element[0]->type != REDIS_REPLY_STRING || + reply->element[0]->len < sizeof("message") - 1) + { + return 0; + } + + /* Get the string/len moving past 'p' if needed */ + off = tolower(reply->element[0]->str[0]) == 'p'; + str = reply->element[0]->str + off; + len = reply->element[0]->len - off; + + return !strncasecmp(str, "subscribe", len) || + !strncasecmp(str, "message", len); + } void redisProcessCallbacks(redisAsyncContext *ac) { @@ -427,7 +511,7 @@ void redisProcessCallbacks(redisAsyncContext *ac) { if (reply == NULL) { /* When the connection is being disconnected and there are * no more replies, this is the cue to really disconnect. */ - if (c->flags & REDIS_DISCONNECTING && sdslen(c->obuf) == 0 + if (c->flags & REDIS_DISCONNECTING && hi_sdslen(c->obuf) == 0 && ac->replies.head == NULL) { __redisAsyncDisconnect(ac); return; @@ -443,8 +527,18 @@ void redisProcessCallbacks(redisAsyncContext *ac) { break; } - /* Even if the context is subscribed, pending regular callbacks will - * get a reply before pub/sub messages arrive. */ + /* Send any non-subscribe related PUSH messages to our PUSH handler + * while allowing subscribe related PUSH messages to pass through. + * This allows existing code to be backward compatible and work in + * either RESP2 or RESP3 mode. */ + if (redisIsSpontaneousPushReply(reply)) { + __redisRunPushCallback(ac, reply); + c->reader->fn->freeObject(reply); + continue; + } + + /* Even if the context is subscribed, pending regular + * callbacks will get a reply before pub/sub messages arrive. */ if (__redisShiftCallback(&ac->replies,&cb) != REDIS_OK) { /* * A spontaneous reply in a not-subscribed context can be the error @@ -497,20 +591,31 @@ void redisProcessCallbacks(redisAsyncContext *ac) { __redisAsyncDisconnect(ac); } +static void __redisAsyncHandleConnectFailure(redisAsyncContext *ac) { + if (ac->onConnect) ac->onConnect(ac, REDIS_ERR); + __redisAsyncDisconnect(ac); +} + /* Internal helper function to detect socket status the first time a read or * write event fires. When connecting was not successful, the connect callback * is called with a REDIS_ERR status and the context is free'd. */ static int __redisAsyncHandleConnect(redisAsyncContext *ac) { int completed = 0; redisContext *c = &(ac->c); + if (redisCheckConnectDone(c, &completed) == REDIS_ERR) { /* Error! */ redisCheckSocketError(c); - if (ac->onConnect) ac->onConnect(ac, REDIS_ERR); - __redisAsyncDisconnect(ac); + __redisAsyncHandleConnectFailure(ac); return REDIS_ERR; } else if (completed == 1) { /* connected! */ + if (c->connection_type == REDIS_CONN_TCP && + redisSetTcpNoDelay(c) == REDIS_ERR) { + __redisAsyncHandleConnectFailure(ac); + return REDIS_ERR; + } + if (ac->onConnect) ac->onConnect(ac, REDIS_OK); c->flags |= REDIS_CONNECTED; return REDIS_OK; @@ -582,8 +687,6 @@ void redisAsyncHandleWrite(redisAsyncContext *ac) { c->funcs->async_write(ac); } -void __redisSetError(redisContext *c, int type, const char *str); - void redisAsyncHandleTimeout(redisAsyncContext *ac) { redisContext *c = &(ac->c); redisCallback cb; @@ -641,7 +744,7 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void const char *cstr, *astr; size_t clen, alen; const char *p; - sds sname; + hisds sname; int ret; /* Don't accept new commands when the connection is about to be closed. */ @@ -665,7 +768,10 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void /* Add every channel/pattern to the list of subscription callbacks. */ while ((p = nextArgument(p,&astr,&alen)) != NULL) { - sname = sdsnewlen(astr,alen); + sname = hi_sdsnewlen(astr,alen); + if (sname == NULL) + goto oom; + if (pvariant) cbdict = ac->sub.patterns; else @@ -680,7 +786,7 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void ret = dictReplace(cbdict,sname,&cb); - if (ret == 0) sdsfree(sname); + if (ret == 0) hi_sdsfree(sname); } } else if (strncasecmp(cstr,"unsubscribe\r\n",13) == 0) { /* It is only useful to call (P)UNSUBSCRIBE when the context is @@ -709,6 +815,9 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void _EL_ADD_WRITE(ac); return REDIS_OK; +oom: + __redisSetError(&(ac->c), REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; } int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap) { @@ -722,7 +831,7 @@ int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdat return REDIS_ERR; status = __redisAsyncCommand(ac,fn,privdata,cmd,len); - free(cmd); + hi_free(cmd); return status; } @@ -736,14 +845,14 @@ int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata } int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen) { - sds cmd; + hisds cmd; int len; int status; len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); if (len < 0) return REDIS_ERR; status = __redisAsyncCommand(ac,fn,privdata,cmd,len); - sdsfree(cmd); + hi_sdsfree(cmd); return status; } @@ -752,15 +861,27 @@ int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void return status; } -void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv) { - if (!ac->c.timeout) { - ac->c.timeout = calloc(1, sizeof(tv)); - } - - if (tv.tv_sec == ac->c.timeout->tv_sec && - tv.tv_usec == ac->c.timeout->tv_usec) { - return; - } - - *ac->c.timeout = tv; +redisAsyncPushFn *redisAsyncSetPushCallback(redisAsyncContext *ac, redisAsyncPushFn *fn) { + redisAsyncPushFn *old = ac->push_cb; + ac->push_cb = fn; + return old; +} + +int redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv) { + if (!ac->c.command_timeout) { + ac->c.command_timeout = hi_calloc(1, sizeof(tv)); + if (ac->c.command_timeout == NULL) { + __redisSetError(&ac->c, REDIS_ERR_OOM, "Out of memory"); + __redisAsyncCopyError(ac); + return REDIS_ERR; + } + } + + if (tv.tv_sec != ac->c.command_timeout->tv_sec || + tv.tv_usec != ac->c.command_timeout->tv_usec) + { + *ac->c.command_timeout = tv; + } + + return REDIS_OK; } diff --git a/deps/hiredis/async.h b/deps/hiredis/async.h index 4f6b3b783..b1d2cb263 100644 --- a/deps/hiredis/async.h +++ b/deps/hiredis/async.h @@ -70,6 +70,7 @@ typedef struct redisAsyncContext { /* Not used by hiredis */ void *data; + void (*dataCleanup)(void *privdata); /* Event library data and hooks */ struct { @@ -105,6 +106,9 @@ typedef struct redisAsyncContext { struct dict *channels; struct dict *patterns; } sub; + + /* Any configured RESP3 PUSH handler */ + redisAsyncPushFn *push_cb; } redisAsyncContext; /* Functions that proxy to hiredis */ @@ -117,7 +121,8 @@ redisAsyncContext *redisAsyncConnectUnix(const char *path); int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn); int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); -void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv); +redisAsyncPushFn *redisAsyncSetPushCallback(redisAsyncContext *ac, redisAsyncPushFn *fn); +int redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv); void redisAsyncDisconnect(redisAsyncContext *ac); void redisAsyncFree(redisAsyncContext *ac); diff --git a/deps/hiredis/async_private.h b/deps/hiredis/async_private.h index d0133ae18..ea0558d42 100644 --- a/deps/hiredis/async_private.h +++ b/deps/hiredis/async_private.h @@ -51,18 +51,21 @@ #define _EL_CLEANUP(ctx) do { \ if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ ctx->ev.cleanup = NULL; \ - } while(0); + } while(0) static inline void refreshTimeout(redisAsyncContext *ctx) { - if (ctx->c.timeout && ctx->ev.scheduleTimer && - (ctx->c.timeout->tv_sec || ctx->c.timeout->tv_usec)) { - ctx->ev.scheduleTimer(ctx->ev.data, *ctx->c.timeout); - // } else { - // printf("Not scheduling timer.. (tmo=%p)\n", ctx->c.timeout); - // if (ctx->c.timeout){ - // printf("tv_sec: %u. tv_usec: %u\n", ctx->c.timeout->tv_sec, - // ctx->c.timeout->tv_usec); - // } + #define REDIS_TIMER_ISSET(tvp) \ + (tvp && ((tvp)->tv_sec || (tvp)->tv_usec)) + + #define REDIS_EL_TIMER(ac, tvp) \ + if ((ac)->ev.scheduleTimer && REDIS_TIMER_ISSET(tvp)) { \ + (ac)->ev.scheduleTimer((ac)->ev.data, *(tvp)); \ + } + + if (ctx->c.flags & REDIS_CONNECTED) { + REDIS_EL_TIMER(ctx, ctx->c.command_timeout); + } else { + REDIS_EL_TIMER(ctx, ctx->c.connect_timeout); } } diff --git a/deps/hiredis/dict.c b/deps/hiredis/dict.c index e17a62546..34a33ead9 100644 --- a/deps/hiredis/dict.c +++ b/deps/hiredis/dict.c @@ -34,6 +34,7 @@ */ #include "fmacros.h" +#include "alloc.h" #include #include #include @@ -71,7 +72,10 @@ static void _dictReset(dict *ht) { /* Create a new hash table */ static dict *dictCreate(dictType *type, void *privDataPtr) { - dict *ht = malloc(sizeof(*ht)); + dict *ht = hi_malloc(sizeof(*ht)); + if (ht == NULL) + return NULL; + _dictInit(ht,type,privDataPtr); return ht; } @@ -97,7 +101,9 @@ static int dictExpand(dict *ht, unsigned long size) { _dictInit(&n, ht->type, ht->privdata); n.size = realsize; n.sizemask = realsize-1; - n.table = calloc(realsize,sizeof(dictEntry*)); + n.table = hi_calloc(realsize,sizeof(dictEntry*)); + if (n.table == NULL) + return DICT_ERR; /* Copy all the elements from the old to the new table: * note that if the old hash table is empty ht->size is zero, @@ -124,7 +130,7 @@ static int dictExpand(dict *ht, unsigned long size) { } } assert(ht->used == 0); - free(ht->table); + hi_free(ht->table); /* Remap the new hashtable in the old */ *ht = n; @@ -142,7 +148,10 @@ static int dictAdd(dict *ht, void *key, void *val) { return DICT_ERR; /* Allocates the memory and stores key */ - entry = malloc(sizeof(*entry)); + entry = hi_malloc(sizeof(*entry)); + if (entry == NULL) + return DICT_ERR; + entry->next = ht->table[index]; ht->table[index] = entry; @@ -166,6 +175,9 @@ static int dictReplace(dict *ht, void *key, void *val) { return 1; /* It already exists, get the entry */ entry = dictFind(ht, key); + if (entry == NULL) + return 0; + /* Free the old value and set the new one */ /* Set the new value and free the old one. Note that it is important * to do that in this order, as the value may just be exactly the same @@ -199,7 +211,7 @@ static int dictDelete(dict *ht, const void *key) { dictFreeEntryKey(ht,de); dictFreeEntryVal(ht,de); - free(de); + hi_free(de); ht->used--; return DICT_OK; } @@ -222,13 +234,13 @@ static int _dictClear(dict *ht) { nextHe = he->next; dictFreeEntryKey(ht, he); dictFreeEntryVal(ht, he); - free(he); + hi_free(he); ht->used--; he = nextHe; } } /* Free the table and the allocated cache structure */ - free(ht->table); + hi_free(ht->table); /* Re-initialize the table */ _dictReset(ht); return DICT_OK; /* never fails */ @@ -237,7 +249,7 @@ static int _dictClear(dict *ht) { /* Clear & Release the hash table */ static void dictRelease(dict *ht) { _dictClear(ht); - free(ht); + hi_free(ht); } static dictEntry *dictFind(dict *ht, const void *key) { @@ -256,7 +268,9 @@ static dictEntry *dictFind(dict *ht, const void *key) { } static dictIterator *dictGetIterator(dict *ht) { - dictIterator *iter = malloc(sizeof(*iter)); + dictIterator *iter = hi_malloc(sizeof(*iter)); + if (iter == NULL) + return NULL; iter->ht = ht; iter->index = -1; @@ -286,7 +300,7 @@ static dictEntry *dictNext(dictIterator *iter) { } static void dictReleaseIterator(dictIterator *iter) { - free(iter); + hi_free(iter); } /* ------------------------- private functions ------------------------------ */ @@ -294,7 +308,7 @@ static void dictReleaseIterator(dictIterator *iter) { /* Expand the hash table if needed */ static int _dictExpandIfNeeded(dict *ht) { /* If the hash table is empty expand it to the initial size, - * if the table is "full" dobule its size. */ + * if the table is "full" double its size. */ if (ht->size == 0) return dictExpand(ht, DICT_HT_INITIAL_SIZE); if (ht->used == ht->size) diff --git a/deps/hiredis/examples/CMakeLists.txt b/deps/hiredis/examples/CMakeLists.txt index dd3a313ac..1d5bc56e0 100644 --- a/deps/hiredis/examples/CMakeLists.txt +++ b/deps/hiredis/examples/CMakeLists.txt @@ -44,3 +44,6 @@ ENDIF() ADD_EXECUTABLE(example example.c) TARGET_LINK_LIBRARIES(example hiredis) + +ADD_EXECUTABLE(example-push example-push.c) +TARGET_LINK_LIBRARIES(example-push hiredis) diff --git a/deps/hiredis/examples/example-ivykis.c b/deps/hiredis/examples/example-ivykis.c index 67affcef3..f57dc3887 100644 --- a/deps/hiredis/examples/example-ivykis.c +++ b/deps/hiredis/examples/example-ivykis.c @@ -33,7 +33,9 @@ void disconnectCallback(const redisAsyncContext *c, int status) { } int main (int argc, char **argv) { +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); +#endif iv_init(); diff --git a/deps/hiredis/examples/example-libev.c b/deps/hiredis/examples/example-libev.c index cc8b166ec..ec474306b 100644 --- a/deps/hiredis/examples/example-libev.c +++ b/deps/hiredis/examples/example-libev.c @@ -33,7 +33,9 @@ void disconnectCallback(const redisAsyncContext *c, int status) { } int main (int argc, char **argv) { +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); +#endif redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); if (c->err) { diff --git a/deps/hiredis/examples/example-libevent-ssl.c b/deps/hiredis/examples/example-libevent-ssl.c index 1021113b9..7d99af1ba 100644 --- a/deps/hiredis/examples/example-libevent-ssl.c +++ b/deps/hiredis/examples/example-libevent-ssl.c @@ -34,7 +34,10 @@ void disconnectCallback(const redisAsyncContext *c, int status) { } int main (int argc, char **argv) { +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); +#endif + struct event_base *base = event_base_new(); if (argc < 5) { fprintf(stderr, @@ -52,13 +55,25 @@ int main (int argc, char **argv) { const char *certKey = argv[5]; const char *caCert = argc > 5 ? argv[6] : NULL; + redisSSLContext *ssl; + redisSSLContextError ssl_error; + + redisInitOpenSSL(); + + ssl = redisCreateSSLContext(caCert, NULL, + cert, certKey, NULL, &ssl_error); + if (!ssl) { + printf("Error: %s\n", redisSSLContextGetError(ssl_error)); + return 1; + } + redisAsyncContext *c = redisAsyncConnect(hostname, port); if (c->err) { /* Let *c leak for now... */ printf("Error: %s\n", c->errstr); return 1; } - if (redisSecureConnection(&c->c, caCert, cert, certKey, "sni") != REDIS_OK) { + if (redisInitiateSSLWithContext(&c->c, ssl) != REDIS_OK) { printf("SSL Error!\n"); exit(1); } @@ -69,5 +84,7 @@ int main (int argc, char **argv) { redisAsyncCommand(c, NULL, NULL, "SET key %b", value, nvalue); redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); event_base_dispatch(base); + + redisFreeSSLContext(ssl); return 0; } diff --git a/deps/hiredis/examples/example-libevent.c b/deps/hiredis/examples/example-libevent.c index 1fe71ae4e..49bddd0c2 100644 --- a/deps/hiredis/examples/example-libevent.c +++ b/deps/hiredis/examples/example-libevent.c @@ -38,13 +38,16 @@ void disconnectCallback(const redisAsyncContext *c, int status) { } int main (int argc, char **argv) { +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); +#endif + struct event_base *base = event_base_new(); redisOptions options = {0}; REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); struct timeval tv = {0}; tv.tv_sec = 1; - options.timeout = &tv; + options.connect_timeout = &tv; redisAsyncContext *c = redisAsyncConnectWithOptions(&options); diff --git a/deps/hiredis/examples/example-libuv.c b/deps/hiredis/examples/example-libuv.c index a5462d410..cbde452b9 100644 --- a/deps/hiredis/examples/example-libuv.c +++ b/deps/hiredis/examples/example-libuv.c @@ -33,7 +33,10 @@ void disconnectCallback(const redisAsyncContext *c, int status) { } int main (int argc, char **argv) { +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); +#endif + uv_loop_t* loop = uv_default_loop(); redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); diff --git a/deps/hiredis/examples/example-push.c b/deps/hiredis/examples/example-push.c new file mode 100644 index 000000000..2d4ab4dc0 --- /dev/null +++ b/deps/hiredis/examples/example-push.c @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2020, Michael Grunder + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include + +#define KEY_COUNT 5 + +#define panicAbort(fmt, ...) \ + do { \ + fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, __LINE__, __func__, __VA_ARGS__); \ + exit(-1); \ + } while (0) + +static void assertReplyAndFree(redisContext *context, redisReply *reply, int type) { + if (reply == NULL) + panicAbort("NULL reply from server (error: %s)", context->errstr); + + if (reply->type != type) { + if (reply->type == REDIS_REPLY_ERROR) + fprintf(stderr, "Redis Error: %s\n", reply->str); + + panicAbort("Expected reply type %d but got type %d", type, reply->type); + } + + freeReplyObject(reply); +} + +/* Switch to the RESP3 protocol and enable client tracking */ +static void enableClientTracking(redisContext *c) { + redisReply *reply = redisCommand(c, "HELLO 3"); + if (reply == NULL || c->err) { + panicAbort("NULL reply or server error (error: %s)", c->errstr); + } + + if (reply->type != REDIS_REPLY_MAP) { + fprintf(stderr, "Error: Can't send HELLO 3 command. Are you sure you're "); + fprintf(stderr, "connected to redis-server >= 6.0.0?\nRedis error: %s\n", + reply->type == REDIS_REPLY_ERROR ? reply->str : "(unknown)"); + exit(-1); + } + + freeReplyObject(reply); + + /* Enable client tracking */ + reply = redisCommand(c, "CLIENT TRACKING ON"); + assertReplyAndFree(c, reply, REDIS_REPLY_STATUS); +} + +void pushReplyHandler(void *privdata, void *r) { + redisReply *reply = r; + int *invalidations = privdata; + + /* Sanity check on the invalidation reply */ + if (reply->type != REDIS_REPLY_PUSH || reply->elements != 2 || + reply->element[1]->type != REDIS_REPLY_ARRAY || + reply->element[1]->element[0]->type != REDIS_REPLY_STRING) + { + panicAbort("%s", "Can't parse PUSH message!"); + } + + /* Increment our invalidation count */ + *invalidations += 1; + + printf("pushReplyHandler(): INVALIDATE '%s' (invalidation count: %d)\n", + reply->element[1]->element[0]->str, *invalidations); + + freeReplyObject(reply); +} + +/* We aren't actually freeing anything here, but it is included to show that we can + * have hiredis call our data destructor when freeing the context */ +void privdata_dtor(void *privdata) { + unsigned int *icount = privdata; + printf("privdata_dtor(): In context privdata dtor (invalidations: %u)\n", *icount); +} + +int main(int argc, char **argv) { + unsigned int j, invalidations = 0; + redisContext *c; + redisReply *reply; + + const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + int port = (argc > 2) ? atoi(argv[2]) : 6379; + + redisOptions o = {0}; + REDIS_OPTIONS_SET_TCP(&o, hostname, port); + + /* Set our context privdata to the address of our invalidation counter. Each + * time our PUSH handler is called, hiredis will pass the privdata for context. + * + * This could also be done after we create the context like so: + * + * c->privdata = &invalidations; + * c->free_privdata = privdata_dtor; + */ + REDIS_OPTIONS_SET_PRIVDATA(&o, &invalidations, privdata_dtor); + + /* Set our custom PUSH message handler */ + o.push_cb = pushReplyHandler; + + c = redisConnectWithOptions(&o); + if (c == NULL || c->err) + panicAbort("Connection error: %s", c ? c->errstr : "OOM"); + + /* Enable RESP3 and turn on client tracking */ + enableClientTracking(c); + + /* Set some keys and then read them back. Once we do that, Redis will deliver + * invalidation push messages whenever the key is modified */ + for (j = 0; j < KEY_COUNT; j++) { + reply = redisCommand(c, "SET key:%d initial:%d", j, j); + assertReplyAndFree(c, reply, REDIS_REPLY_STATUS); + + reply = redisCommand(c, "GET key:%d", j); + assertReplyAndFree(c, reply, REDIS_REPLY_STRING); + } + + /* Trigger invalidation messages by updating keys we just read */ + for (j = 0; j < KEY_COUNT; j++) { + printf(" main(): SET key:%d update:%d\n", j, j); + reply = redisCommand(c, "SET key:%d update:%d", j, j); + assertReplyAndFree(c, reply, REDIS_REPLY_STATUS); + printf(" main(): SET REPLY OK\n"); + } + + printf("\nTotal detected invalidations: %d, expected: %d\n", invalidations, KEY_COUNT); + + /* PING server */ + redisFree(c); +} diff --git a/deps/hiredis/examples/example-ssl.c b/deps/hiredis/examples/example-ssl.c index 81f4648c6..c754177cf 100644 --- a/deps/hiredis/examples/example-ssl.c +++ b/deps/hiredis/examples/example-ssl.c @@ -4,9 +4,12 @@ #include #include +#include int main(int argc, char **argv) { unsigned int j; + redisSSLContext *ssl; + redisSSLContextError ssl_error; redisContext *c; redisReply *reply; if (argc < 4) { @@ -19,10 +22,18 @@ int main(int argc, char **argv) { const char *key = argv[4]; const char *ca = argc > 4 ? argv[5] : NULL; + redisInitOpenSSL(); + ssl = redisCreateSSLContext(ca, NULL, cert, key, NULL, &ssl_error); + if (!ssl) { + printf("SSL Context error: %s\n", + redisSSLContextGetError(ssl_error)); + exit(1); + } + struct timeval tv = { 1, 500000 }; // 1.5 seconds redisOptions options = {0}; REDIS_OPTIONS_SET_TCP(&options, hostname, port); - options.timeout = &tv; + options.connect_timeout = &tv; c = redisConnectWithOptions(&options); if (c == NULL || c->err) { @@ -35,7 +46,7 @@ int main(int argc, char **argv) { exit(1); } - if (redisSecureConnection(c, ca, cert, key, "sni") != REDIS_OK) { + if (redisInitiateSSLWithContext(c, ssl) != REDIS_OK) { printf("Couldn't initialize SSL!\n"); printf("Error: %s\n", c->errstr); redisFree(c); @@ -93,5 +104,7 @@ int main(int argc, char **argv) { /* Disconnects and frees the context */ redisFree(c); + redisFreeSSLContext(ssl); + return 0; } diff --git a/deps/hiredis/examples/example.c b/deps/hiredis/examples/example.c index 0e93fc8b3..15dacbd18 100644 --- a/deps/hiredis/examples/example.c +++ b/deps/hiredis/examples/example.c @@ -1,8 +1,8 @@ #include #include #include - #include +#include int main(int argc, char **argv) { unsigned int j, isunix = 0; diff --git a/deps/hiredis/hiredis-config.cmake.in b/deps/hiredis/hiredis-config.cmake.in new file mode 100644 index 000000000..98851dcee --- /dev/null +++ b/deps/hiredis/hiredis-config.cmake.in @@ -0,0 +1,13 @@ +@PACKAGE_INIT@ + +set_and_check(hiredis_INCLUDEDIR "@PACKAGE_INCLUDE_INSTALL_DIR@") + +IF (NOT TARGET hiredis::hiredis) + INCLUDE(${CMAKE_CURRENT_LIST_DIR}/hiredis-targets.cmake) +ENDIF() + +SET(hiredis_LIBRARIES hiredis::hiredis) +SET(hiredis_INCLUDE_DIRS ${hiredis_INCLUDEDIR}) + +check_required_components(hiredis) + diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index abd94c01d..51f22a665 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -44,8 +44,11 @@ #include "async.h" #include "win32.h" +extern int redisContextUpdateConnectTimeout(redisContext *c, const struct timeval *timeout); +extern int redisContextUpdateCommandTimeout(redisContext *c, const struct timeval *timeout); + static redisContextFuncs redisContextDefaultFuncs = { - .free_privdata = NULL, + .free_privctx = NULL, .async_read = redisAsyncRead, .async_write = redisAsyncWrite, .read = redisNetRead, @@ -74,7 +77,7 @@ static redisReplyObjectFunctions defaultFunctions = { /* Create a reply object */ static redisReply *createReplyObject(int type) { - redisReply *r = calloc(1,sizeof(*r)); + redisReply *r = hi_calloc(1,sizeof(*r)); if (r == NULL) return NULL; @@ -97,20 +100,22 @@ void freeReplyObject(void *reply) { case REDIS_REPLY_ARRAY: case REDIS_REPLY_MAP: case REDIS_REPLY_SET: + case REDIS_REPLY_PUSH: if (r->element != NULL) { for (j = 0; j < r->elements; j++) freeReplyObject(r->element[j]); - free(r->element); + hi_free(r->element); } break; case REDIS_REPLY_ERROR: case REDIS_REPLY_STATUS: case REDIS_REPLY_STRING: case REDIS_REPLY_DOUBLE: - free(r->str); + case REDIS_REPLY_VERB: + hi_free(r->str); break; } - free(r); + hi_free(r); } static void *createStringObject(const redisReadTask *task, char *str, size_t len) { @@ -128,22 +133,18 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len /* Copy string value */ if (task->type == REDIS_REPLY_VERB) { - buf = malloc(len-4+1); /* Skip 4 bytes of verbatim type header. */ - if (buf == NULL) { - freeReplyObject(r); - return NULL; - } + buf = hi_malloc(len-4+1); /* Skip 4 bytes of verbatim type header. */ + if (buf == NULL) goto oom; + memcpy(r->vtype,str,3); r->vtype[3] = '\0'; memcpy(buf,str+4,len-4); buf[len-4] = '\0'; - r->len = len-4; + r->len = len - 4; } else { - buf = malloc(len+1); - if (buf == NULL) { - freeReplyObject(r); - return NULL; - } + buf = hi_malloc(len+1); + if (buf == NULL) goto oom; + memcpy(buf,str,len); buf[len] = '\0'; r->len = len; @@ -154,10 +155,15 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len parent = task->parent->obj; assert(parent->type == REDIS_REPLY_ARRAY || parent->type == REDIS_REPLY_MAP || - parent->type == REDIS_REPLY_SET); + parent->type == REDIS_REPLY_SET || + parent->type == REDIS_REPLY_PUSH); parent->element[task->idx] = r; } return r; + +oom: + freeReplyObject(r); + return NULL; } static void *createArrayObject(const redisReadTask *task, size_t elements) { @@ -168,7 +174,7 @@ static void *createArrayObject(const redisReadTask *task, size_t elements) { return NULL; if (elements > 0) { - r->element = calloc(elements,sizeof(redisReply*)); + r->element = hi_calloc(elements,sizeof(redisReply*)); if (r->element == NULL) { freeReplyObject(r); return NULL; @@ -181,7 +187,8 @@ static void *createArrayObject(const redisReadTask *task, size_t elements) { parent = task->parent->obj; assert(parent->type == REDIS_REPLY_ARRAY || parent->type == REDIS_REPLY_MAP || - parent->type == REDIS_REPLY_SET); + parent->type == REDIS_REPLY_SET || + parent->type == REDIS_REPLY_PUSH); parent->element[task->idx] = r; } return r; @@ -200,7 +207,8 @@ static void *createIntegerObject(const redisReadTask *task, long long value) { parent = task->parent->obj; assert(parent->type == REDIS_REPLY_ARRAY || parent->type == REDIS_REPLY_MAP || - parent->type == REDIS_REPLY_SET); + parent->type == REDIS_REPLY_SET || + parent->type == REDIS_REPLY_PUSH); parent->element[task->idx] = r; } return r; @@ -214,7 +222,7 @@ static void *createDoubleObject(const redisReadTask *task, double value, char *s return NULL; r->dval = value; - r->str = malloc(len+1); + r->str = hi_malloc(len+1); if (r->str == NULL) { freeReplyObject(r); return NULL; @@ -249,7 +257,8 @@ static void *createNilObject(const redisReadTask *task) { parent = task->parent->obj; assert(parent->type == REDIS_REPLY_ARRAY || parent->type == REDIS_REPLY_MAP || - parent->type == REDIS_REPLY_SET); + parent->type == REDIS_REPLY_SET || + parent->type == REDIS_REPLY_PUSH); parent->element[task->idx] = r; } return r; @@ -297,7 +306,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { const char *c = format; char *cmd = NULL; /* final command */ int pos; /* position in final command */ - sds curarg, newarg; /* current argument */ + hisds curarg, newarg; /* current argument */ int touched = 0; /* was the current argument touched? */ char **curargv = NULL, **newargv = NULL; int argc = 0; @@ -310,7 +319,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { return -1; /* Build the command string accordingly to protocol */ - curarg = sdsempty(); + curarg = hi_sdsempty(); if (curarg == NULL) return -1; @@ -318,19 +327,19 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { if (*c != '%' || c[1] == '\0') { if (*c == ' ') { if (touched) { - newargv = realloc(curargv,sizeof(char*)*(argc+1)); + newargv = hi_realloc(curargv,sizeof(char*)*(argc+1)); if (newargv == NULL) goto memory_err; curargv = newargv; curargv[argc++] = curarg; - totlen += bulklen(sdslen(curarg)); + totlen += bulklen(hi_sdslen(curarg)); /* curarg is put in argv so it can be overwritten. */ - curarg = sdsempty(); + curarg = hi_sdsempty(); if (curarg == NULL) goto memory_err; touched = 0; } } else { - newarg = sdscatlen(curarg,c,1); + newarg = hi_sdscatlen(curarg,c,1); if (newarg == NULL) goto memory_err; curarg = newarg; touched = 1; @@ -347,16 +356,16 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { arg = va_arg(ap,char*); size = strlen(arg); if (size > 0) - newarg = sdscatlen(curarg,arg,size); + newarg = hi_sdscatlen(curarg,arg,size); break; case 'b': arg = va_arg(ap,char*); size = va_arg(ap,size_t); if (size > 0) - newarg = sdscatlen(curarg,arg,size); + newarg = hi_sdscatlen(curarg,arg,size); break; case '%': - newarg = sdscat(curarg,"%"); + newarg = hi_sdscat(curarg,"%"); break; default: /* Try to detect printf format */ @@ -444,7 +453,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { if (_l < sizeof(_format)-2) { memcpy(_format,c,_l); _format[_l] = '\0'; - newarg = sdscatvprintf(curarg,_format,_cpy); + newarg = hi_sdscatvprintf(curarg,_format,_cpy); /* Update current position (note: outer blocks * increment c twice so compensate here) */ @@ -467,13 +476,13 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { /* Add the last argument if needed */ if (touched) { - newargv = realloc(curargv,sizeof(char*)*(argc+1)); + newargv = hi_realloc(curargv,sizeof(char*)*(argc+1)); if (newargv == NULL) goto memory_err; curargv = newargv; curargv[argc++] = curarg; - totlen += bulklen(sdslen(curarg)); + totlen += bulklen(hi_sdslen(curarg)); } else { - sdsfree(curarg); + hi_sdsfree(curarg); } /* Clear curarg because it was put in curargv or was free'd. */ @@ -483,22 +492,22 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { totlen += 1+countDigits(argc)+2; /* Build the command at protocol level */ - cmd = malloc(totlen+1); + cmd = hi_malloc(totlen+1); if (cmd == NULL) goto memory_err; pos = sprintf(cmd,"*%d\r\n",argc); for (j = 0; j < argc; j++) { - pos += sprintf(cmd+pos,"$%zu\r\n",sdslen(curargv[j])); - memcpy(cmd+pos,curargv[j],sdslen(curargv[j])); - pos += sdslen(curargv[j]); - sdsfree(curargv[j]); + pos += sprintf(cmd+pos,"$%zu\r\n",hi_sdslen(curargv[j])); + memcpy(cmd+pos,curargv[j],hi_sdslen(curargv[j])); + pos += hi_sdslen(curargv[j]); + hi_sdsfree(curargv[j]); cmd[pos++] = '\r'; cmd[pos++] = '\n'; } assert(pos == totlen); cmd[pos] = '\0'; - free(curargv); + hi_free(curargv); *target = cmd; return totlen; @@ -513,12 +522,12 @@ memory_err: cleanup: if (curargv) { while(argc--) - sdsfree(curargv[argc]); - free(curargv); + hi_sdsfree(curargv[argc]); + hi_free(curargv); } - sdsfree(curarg); - free(cmd); + hi_sdsfree(curarg); + hi_free(cmd); return error_type; } @@ -550,16 +559,16 @@ int redisFormatCommand(char **target, const char *format, ...) { return len; } -/* Format a command according to the Redis protocol using an sds string and - * sdscatfmt for the processing of arguments. This function takes the +/* Format a command according to the Redis protocol using an hisds string and + * hi_sdscatfmt for the processing of arguments. This function takes the * number of arguments, an array with arguments and an array with their * lengths. If the latter is set to NULL, strlen will be used to compute the * argument lengths. */ -int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv, +int redisFormatSdsCommandArgv(hisds *target, int argc, const char **argv, const size_t *argvlen) { - sds cmd; + hisds cmd, aux; unsigned long long totlen; int j; size_t len; @@ -576,32 +585,36 @@ int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv, } /* Use an SDS string for command construction */ - cmd = sdsempty(); + cmd = hi_sdsempty(); if (cmd == NULL) return -1; /* We already know how much storage we need */ - cmd = sdsMakeRoomFor(cmd, totlen); - if (cmd == NULL) + aux = hi_sdsMakeRoomFor(cmd, totlen); + if (aux == NULL) { + hi_sdsfree(cmd); return -1; - - /* Construct command */ - cmd = sdscatfmt(cmd, "*%i\r\n", argc); - for (j=0; j < argc; j++) { - len = argvlen ? argvlen[j] : strlen(argv[j]); - cmd = sdscatfmt(cmd, "$%u\r\n", len); - cmd = sdscatlen(cmd, argv[j], len); - cmd = sdscatlen(cmd, "\r\n", sizeof("\r\n")-1); } - assert(sdslen(cmd)==totlen); + cmd = aux; + + /* Construct command */ + cmd = hi_sdscatfmt(cmd, "*%i\r\n", argc); + for (j=0; j < argc; j++) { + len = argvlen ? argvlen[j] : strlen(argv[j]); + cmd = hi_sdscatfmt(cmd, "$%u\r\n", len); + cmd = hi_sdscatlen(cmd, argv[j], len); + cmd = hi_sdscatlen(cmd, "\r\n", sizeof("\r\n")-1); + } + + assert(hi_sdslen(cmd)==totlen); *target = cmd; return totlen; } -void redisFreeSdsCommand(sds cmd) { - sdsfree(cmd); +void redisFreeSdsCommand(hisds cmd) { + hi_sdsfree(cmd); } /* Format a command according to the Redis protocol. This function takes the @@ -627,7 +640,7 @@ int redisFormatCommandArgv(char **target, int argc, const char **argv, const siz } /* Build the command at protocol level */ - cmd = malloc(totlen+1); + cmd = hi_malloc(totlen+1); if (cmd == NULL) return -1; @@ -648,7 +661,7 @@ int redisFormatCommandArgv(char **target, int argc, const char **argv, const siz } void redisFreeCommand(char *cmd) { - free(cmd); + hi_free(cmd); } void __redisSetError(redisContext *c, int type, const char *str) { @@ -671,15 +684,21 @@ redisReader *redisReaderCreate(void) { return redisReaderCreateWithFunctions(&defaultFunctions); } -static redisContext *redisContextInit(const redisOptions *options) { +static void redisPushAutoFree(void *privdata, void *reply) { + (void)privdata; + freeReplyObject(reply); +} + +static redisContext *redisContextInit(void) { redisContext *c; - c = calloc(1, sizeof(*c)); + c = hi_calloc(1, sizeof(*c)); if (c == NULL) return NULL; c->funcs = &redisContextDefaultFuncs; - c->obuf = sdsempty(); + + c->obuf = hi_sdsempty(); c->reader = redisReaderCreate(); c->fd = REDIS_INVALID_FD; @@ -687,7 +706,7 @@ static redisContext *redisContextInit(const redisOptions *options) { redisFree(c); return NULL; } - (void)options; /* options are used in other functions */ + return c; } @@ -696,18 +715,23 @@ void redisFree(redisContext *c) { return; redisNetClose(c); - sdsfree(c->obuf); + hi_sdsfree(c->obuf); redisReaderFree(c->reader); - free(c->tcp.host); - free(c->tcp.source_addr); - free(c->unix_sock.path); - free(c->timeout); - free(c->saddr); - if (c->funcs->free_privdata) { - c->funcs->free_privdata(c->privdata); - } + hi_free(c->tcp.host); + hi_free(c->tcp.source_addr); + hi_free(c->unix_sock.path); + hi_free(c->connect_timeout); + hi_free(c->command_timeout); + hi_free(c->saddr); + + if (c->privdata && c->free_privdata) + c->free_privdata(c->privdata); + + if (c->funcs->free_privctx) + c->funcs->free_privctx(c->privctx); + memset(c, 0xff, sizeof(*c)); - free(c); + hi_free(c); } redisFD redisFreeKeepFd(redisContext *c) { @@ -721,35 +745,46 @@ int redisReconnect(redisContext *c) { c->err = 0; memset(c->errstr, '\0', strlen(c->errstr)); - if (c->privdata && c->funcs->free_privdata) { - c->funcs->free_privdata(c->privdata); - c->privdata = NULL; + if (c->privctx && c->funcs->free_privctx) { + c->funcs->free_privctx(c->privctx); + c->privctx = NULL; } redisNetClose(c); - sdsfree(c->obuf); + hi_sdsfree(c->obuf); redisReaderFree(c->reader); - c->obuf = sdsempty(); + c->obuf = hi_sdsempty(); c->reader = redisReaderCreate(); + if (c->obuf == NULL || c->reader == NULL) { + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; + } + + int ret = REDIS_ERR; if (c->connection_type == REDIS_CONN_TCP) { - return redisContextConnectBindTcp(c, c->tcp.host, c->tcp.port, - c->timeout, c->tcp.source_addr); + ret = redisContextConnectBindTcp(c, c->tcp.host, c->tcp.port, + c->connect_timeout, c->tcp.source_addr); } else if (c->connection_type == REDIS_CONN_UNIX) { - return redisContextConnectUnix(c, c->unix_sock.path, c->timeout); + ret = redisContextConnectUnix(c, c->unix_sock.path, c->connect_timeout); } else { /* Something bad happened here and shouldn't have. There isn't enough information in the context to reconnect. */ __redisSetError(c,REDIS_ERR_OTHER,"Not enough information to reconnect"); + ret = REDIS_ERR; } - return REDIS_ERR; + if (c->command_timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) { + redisContextSetTimeout(c, *c->command_timeout); + } + + return ret; } redisContext *redisConnectWithOptions(const redisOptions *options) { - redisContext *c = redisContextInit(options); + redisContext *c = redisContextInit(); if (c == NULL) { return NULL; } @@ -760,16 +795,32 @@ redisContext *redisConnectWithOptions(const redisOptions *options) { c->flags |= REDIS_REUSEADDR; } if (options->options & REDIS_OPT_NOAUTOFREE) { - c->flags |= REDIS_NO_AUTO_FREE; + c->flags |= REDIS_NO_AUTO_FREE; + } + + /* Set any user supplied RESP3 PUSH handler or use freeReplyObject + * as a default unless specifically flagged that we don't want one. */ + if (options->push_cb != NULL) + redisSetPushCallback(c, options->push_cb); + else if (!(options->options & REDIS_OPT_NO_PUSH_AUTOFREE)) + redisSetPushCallback(c, redisPushAutoFree); + + c->privdata = options->privdata; + c->free_privdata = options->free_privdata; + + if (redisContextUpdateConnectTimeout(c, options->connect_timeout) != REDIS_OK || + redisContextUpdateCommandTimeout(c, options->command_timeout) != REDIS_OK) { + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return c; } if (options->type == REDIS_CONN_TCP) { redisContextConnectBindTcp(c, options->endpoint.tcp.ip, - options->endpoint.tcp.port, options->timeout, + options->endpoint.tcp.port, options->connect_timeout, options->endpoint.tcp.source_addr); } else if (options->type == REDIS_CONN_UNIX) { redisContextConnectUnix(c, options->endpoint.unix_socket, - options->timeout); + options->connect_timeout); } else if (options->type == REDIS_CONN_USERFD) { c->fd = options->endpoint.fd; c->flags |= REDIS_CONNECTED; @@ -777,9 +828,11 @@ redisContext *redisConnectWithOptions(const redisOptions *options) { // Unknown type - FIXME - FREE return NULL; } - if (options->timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) { - redisContextSetTimeout(c, *options->timeout); + + if (options->command_timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) { + redisContextSetTimeout(c, *options->command_timeout); } + return c; } @@ -795,7 +848,7 @@ redisContext *redisConnect(const char *ip, int port) { redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) { redisOptions options = {0}; REDIS_OPTIONS_SET_TCP(&options, ip, port); - options.timeout = &tv; + options.connect_timeout = &tv; return redisConnectWithOptions(&options); } @@ -833,7 +886,7 @@ redisContext *redisConnectUnix(const char *path) { redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) { redisOptions options = {0}; REDIS_OPTIONS_SET_UNIX(&options, path); - options.timeout = &tv; + options.connect_timeout = &tv; return redisConnectWithOptions(&options); } @@ -865,6 +918,13 @@ int redisEnableKeepAlive(redisContext *c) { return REDIS_OK; } +/* Set a user provided RESP3 PUSH handler and return any old one set. */ +redisPushFn *redisSetPushCallback(redisContext *c, redisPushFn *fn) { + redisPushFn *old = c->push_cb; + c->push_cb = fn; + return old; +} + /* Use this function to handle a read event on the descriptor. It will try * and read some bytes from the socket and feed them to the reply parser. * @@ -906,21 +966,27 @@ int redisBufferWrite(redisContext *c, int *done) { if (c->err) return REDIS_ERR; - if (sdslen(c->obuf) > 0) { - int nwritten = c->funcs->write(c); + if (hi_sdslen(c->obuf) > 0) { + ssize_t nwritten = c->funcs->write(c); if (nwritten < 0) { return REDIS_ERR; } else if (nwritten > 0) { - if (nwritten == (signed)sdslen(c->obuf)) { - sdsfree(c->obuf); - c->obuf = sdsempty(); + if (nwritten == (ssize_t)hi_sdslen(c->obuf)) { + hi_sdsfree(c->obuf); + c->obuf = hi_sdsempty(); + if (c->obuf == NULL) + goto oom; } else { - sdsrange(c->obuf,nwritten,-1); + if (hi_sdsrange(c->obuf,nwritten,-1) < 0) goto oom; } } } - if (done != NULL) *done = (sdslen(c->obuf) == 0); + if (done != NULL) *done = (hi_sdslen(c->obuf) == 0); return REDIS_OK; + +oom: + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; } /* Internal helper function to try and get a reply from the reader, @@ -930,9 +996,21 @@ int redisGetReplyFromReader(redisContext *c, void **reply) { __redisSetError(c,c->reader->err,c->reader->errstr); return REDIS_ERR; } + return REDIS_OK; } +/* Internal helper that returns 1 if the reply was a RESP3 PUSH + * message and we handled it with a user-provided callback. */ +static int redisHandledPushReply(redisContext *c, void *reply) { + if (reply && c->push_cb && redisIsPushReply(reply)) { + c->push_cb(c->privdata, reply); + return 1; + } + + return 0; +} + int redisGetReply(redisContext *c, void **reply) { int wdone = 0; void *aux = NULL; @@ -953,13 +1031,23 @@ int redisGetReply(redisContext *c, void **reply) { do { if (redisBufferRead(c) == REDIS_ERR) return REDIS_ERR; - if (redisGetReplyFromReader(c,&aux) == REDIS_ERR) - return REDIS_ERR; + + /* We loop here in case the user has specified a RESP3 + * PUSH handler (e.g. for client tracking). */ + do { + if (redisGetReplyFromReader(c,&aux) == REDIS_ERR) + return REDIS_ERR; + } while (redisHandledPushReply(c, aux)); } while (aux == NULL); } - /* Set reply object */ - if (reply != NULL) *reply = aux; + /* Set reply or free it if we were passed NULL */ + if (reply != NULL) { + *reply = aux; + } else { + freeReplyObject(aux); + } + return REDIS_OK; } @@ -971,9 +1059,9 @@ int redisGetReply(redisContext *c, void **reply) { * the reply (or replies in pub/sub). */ int __redisAppendCommand(redisContext *c, const char *cmd, size_t len) { - sds newbuf; + hisds newbuf; - newbuf = sdscatlen(c->obuf,cmd,len); + newbuf = hi_sdscatlen(c->obuf,cmd,len); if (newbuf == NULL) { __redisSetError(c,REDIS_ERR_OOM,"Out of memory"); return REDIS_ERR; @@ -1006,11 +1094,11 @@ int redisvAppendCommand(redisContext *c, const char *format, va_list ap) { } if (__redisAppendCommand(c,cmd,len) != REDIS_OK) { - free(cmd); + hi_free(cmd); return REDIS_ERR; } - free(cmd); + hi_free(cmd); return REDIS_OK; } @@ -1025,7 +1113,7 @@ int redisAppendCommand(redisContext *c, const char *format, ...) { } int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) { - sds cmd; + hisds cmd; int len; len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); @@ -1035,11 +1123,11 @@ int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const s } if (__redisAppendCommand(c,cmd,len) != REDIS_OK) { - sdsfree(cmd); + hi_sdsfree(cmd); return REDIS_ERR; } - sdsfree(cmd); + hi_sdsfree(cmd); return REDIS_OK; } diff --git a/deps/hiredis/hiredis.h b/deps/hiredis/hiredis.h index 69dc39c5e..b597394d4 100644 --- a/deps/hiredis/hiredis.h +++ b/deps/hiredis/hiredis.h @@ -39,14 +39,16 @@ #include /* for struct timeval */ #else struct timeval; /* forward declaration */ +typedef long long ssize_t; #endif #include /* uintXX_t, etc */ -#include "sds.h" /* for sds */ +#include "sds.h" /* for hisds */ +#include "alloc.h" /* for allocation wrappers */ -#define HIREDIS_MAJOR 0 -#define HIREDIS_MINOR 14 +#define HIREDIS_MAJOR 1 +#define HIREDIS_MINOR 0 #define HIREDIS_PATCH 0 -#define HIREDIS_SONAME 0.14 +#define HIREDIS_SONAME 1.0.0 /* Connection type can be blocking or non-blocking and is set in the * least significant bit of the flags field in redisContext. */ @@ -90,6 +92,15 @@ struct timeval; /* forward declaration */ * SO_REUSEADDR is being used. */ #define REDIS_CONNECT_RETRIES 10 +/* Forward declarations for structs defined elsewhere */ +struct redisAsyncContext; +struct redisContext; + +/* RESP3 push helpers and callback prototypes */ +#define redisIsPushReply(r) (((redisReply*)(r))->type == REDIS_REPLY_PUSH) +typedef void (redisPushFn)(void *, void *); +typedef void (redisAsyncPushFn)(struct redisAsyncContext *, void *); + #ifdef __cplusplus extern "C" { #endif @@ -101,7 +112,7 @@ typedef struct redisReply { double dval; /* The double when type is REDIS_REPLY_DOUBLE */ size_t len; /* Length of string */ char *str; /* Used for REDIS_REPLY_ERROR, REDIS_REPLY_STRING - and REDIS_REPLY_DOUBLE (in additionl to dval). */ + REDIS_REPLY_VERB, and REDIS_REPLY_DOUBLE (in additional to dval). */ char vtype[4]; /* Used for REDIS_REPLY_VERB, contains the null terminated 3 character content type, such as "txt". */ size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */ @@ -117,9 +128,9 @@ void freeReplyObject(void *reply); int redisvFormatCommand(char **target, const char *format, va_list ap); int redisFormatCommand(char **target, const char *format, ...); int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen); -int redisFormatSdsCommandArgv(sds *target, int argc, const char ** argv, const size_t *argvlen); +int redisFormatSdsCommandArgv(hisds *target, int argc, const char ** argv, const size_t *argvlen); void redisFreeCommand(char *cmd); -void redisFreeSdsCommand(sds cmd); +void redisFreeSdsCommand(hisds cmd); enum redisConnectionType { REDIS_CONN_TCP, @@ -138,6 +149,9 @@ struct redisSsl; */ #define REDIS_OPT_NOAUTOFREE 0x04 +/* Don't automatically intercept and free RESP3 PUSH replies. */ +#define REDIS_OPT_NO_PUSH_AUTOFREE 0x08 + /* In Unix systems a file descriptor is a regular signed int, with -1 * representing an invalid descriptor. In Windows it is a SOCKET * (32- or 64-bit unsigned integer depending on the architecture), where @@ -162,8 +176,11 @@ typedef struct { int type; /* bit field of REDIS_OPT_xxx */ int options; - /* timeout value. if NULL, no timeout is used */ - const struct timeval *timeout; + /* timeout value for connect operation. If NULL, no timeout is used */ + const struct timeval *connect_timeout; + /* timeout value for commands. If NULL, no timeout is used. This can be + * updated at runtime with redisSetTimeout/redisAsyncSetTimeout. */ + const struct timeval *command_timeout; union { /** use this field for tcp/ip connections */ struct { @@ -178,6 +195,14 @@ typedef struct { * file descriptor */ redisFD fd; } endpoint; + + /* Optional user defined data/destructor */ + void *privdata; + void (*free_privdata)(void *); + + /* A user defined PUSH message callback */ + redisPushFn *push_cb; + redisAsyncPushFn *async_push_cb; } redisOptions; /** @@ -192,15 +217,16 @@ typedef struct { (opts)->type = REDIS_CONN_UNIX; \ (opts)->endpoint.unix_socket = path; -struct redisAsyncContext; -struct redisContext; +#define REDIS_OPTIONS_SET_PRIVDATA(opts, data, dtor) \ + (opts)->privdata = data; \ + (opts)->free_privdata = dtor; \ typedef struct redisContextFuncs { - void (*free_privdata)(void *); + void (*free_privctx)(void *); void (*async_read)(struct redisAsyncContext *); void (*async_write)(struct redisAsyncContext *); - int (*read)(struct redisContext *, char *, size_t); - int (*write)(struct redisContext *); + ssize_t (*read)(struct redisContext *, char *, size_t); + ssize_t (*write)(struct redisContext *); } redisContextFuncs; /* Context for a connection to Redis */ @@ -215,7 +241,8 @@ typedef struct redisContext { redisReader *reader; /* Protocol reader */ enum redisConnectionType connection_type; - struct timeval *timeout; + struct timeval *connect_timeout; + struct timeval *command_timeout; struct { char *host; @@ -231,8 +258,17 @@ typedef struct redisContext { struct sockadr *saddr; size_t addrlen; - /* Additional private data for hiredis addons such as SSL */ + /* Optional data and corresponding destructor users can use to provide + * context to a given redisContext. Not used by hiredis. */ void *privdata; + void (*free_privdata)(void *); + + /* Internal context pointer presently used by hiredis to manage + * SSL connections. */ + void *privctx; + + /* An optional RESP3 PUSH handler */ + redisPushFn *push_cb; } redisContext; redisContext *redisConnectWithOptions(const redisOptions *options); @@ -259,6 +295,7 @@ redisContext *redisConnectFd(redisFD fd); */ int redisReconnect(redisContext *c); +redisPushFn *redisSetPushCallback(redisContext *c, redisPushFn *fn); int redisSetTimeout(redisContext *c, const struct timeval tv); int redisEnableKeepAlive(redisContext *c); void redisFree(redisContext *c); diff --git a/deps/hiredis/hiredis.pc.in b/deps/hiredis/hiredis.pc.in index 140b040f1..91b773183 100644 --- a/deps/hiredis/hiredis.pc.in +++ b/deps/hiredis/hiredis.pc.in @@ -1,6 +1,7 @@ prefix=@CMAKE_INSTALL_PREFIX@ +install_libdir=@CMAKE_INSTALL_LIBDIR@ exec_prefix=${prefix} -libdir=${exec_prefix}/lib +libdir=${exec_prefix}/${install_libdir} includedir=${prefix}/include pkgincludedir=${includedir}/hiredis diff --git a/deps/hiredis/hiredis_ssl-config.cmake.in b/deps/hiredis/hiredis_ssl-config.cmake.in new file mode 100644 index 000000000..9a283dfc2 --- /dev/null +++ b/deps/hiredis/hiredis_ssl-config.cmake.in @@ -0,0 +1,13 @@ +@PACKAGE_INIT@ + +set_and_check(hiredis_ssl_INCLUDEDIR "@PACKAGE_INCLUDE_INSTALL_DIR@") + +IF (NOT TARGET hiredis::hiredis_ssl) + INCLUDE(${CMAKE_CURRENT_LIST_DIR}/hiredis_ssl-targets.cmake) +ENDIF() + +SET(hiredis_ssl_LIBRARIES hiredis::hiredis_ssl) +SET(hiredis_ssl_INCLUDE_DIRS ${hiredis_ssl_INCLUDEDIR}) + +check_required_components(hiredis_ssl) + diff --git a/deps/hiredis/hiredis_ssl.h b/deps/hiredis/hiredis_ssl.h index f844f9548..604efe0c1 100644 --- a/deps/hiredis/hiredis_ssl.h +++ b/deps/hiredis/hiredis_ssl.h @@ -32,22 +32,96 @@ #ifndef __HIREDIS_SSL_H #define __HIREDIS_SSL_H +#ifdef __cplusplus +extern "C" { +#endif + /* This is the underlying struct for SSL in ssl.h, which is not included to * keep build dependencies short here. */ struct ssl_st; -/** - * Secure the connection using SSL. This should be done before any command is - * executed on the connection. +/* A wrapper around OpenSSL SSL_CTX to allow easy SSL use without directly + * calling OpenSSL. */ -int redisSecureConnection(redisContext *c, const char *capath, const char *certpath, - const char *keypath, const char *servername); +typedef struct redisSSLContext redisSSLContext; /** - * Initiate SSL/TLS negotiation on a provided context. + * Initialization errors that redisCreateSSLContext() may return. + */ + +typedef enum { + REDIS_SSL_CTX_NONE = 0, /* No Error */ + REDIS_SSL_CTX_CREATE_FAILED, /* Failed to create OpenSSL SSL_CTX */ + REDIS_SSL_CTX_CERT_KEY_REQUIRED, /* Client cert and key must both be specified or skipped */ + REDIS_SSL_CTX_CA_CERT_LOAD_FAILED, /* Failed to load CA Certificate or CA Path */ + REDIS_SSL_CTX_CLIENT_CERT_LOAD_FAILED, /* Failed to load client certificate */ + REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED /* Failed to load private key */ +} redisSSLContextError; + +/** + * Return the error message corresponding with the specified error code. + */ + +const char *redisSSLContextGetError(redisSSLContextError error); + +/** + * Helper function to initialize the OpenSSL library. + * + * OpenSSL requires one-time initialization before it can be used. Callers should + * call this function only once, and only if OpenSSL is not directly initialized + * elsewhere. + */ +int redisInitOpenSSL(void); + +/** + * Helper function to initialize an OpenSSL context that can be used + * to initiate SSL connections. + * + * cacert_filename is an optional name of a CA certificate/bundle file to load + * and use for validation. + * + * capath is an optional directory path where trusted CA certificate files are + * stored in an OpenSSL-compatible structure. + * + * cert_filename and private_key_filename are optional names of a client side + * certificate and private key files to use for authentication. They need to + * be both specified or omitted. + * + * server_name is an optional and will be used as a server name indication + * (SNI) TLS extension. + * + * If error is non-null, it will be populated in case the context creation fails + * (returning a NULL). + */ + +redisSSLContext *redisCreateSSLContext(const char *cacert_filename, const char *capath, + const char *cert_filename, const char *private_key_filename, + const char *server_name, redisSSLContextError *error); + +/** + * Free a previously created OpenSSL context. + */ +void redisFreeSSLContext(redisSSLContext *redis_ssl_ctx); + +/** + * Initiate SSL on an existing redisContext. + * + * This is similar to redisInitiateSSL() but does not require the caller + * to directly interact with OpenSSL, and instead uses a redisSSLContext + * previously created using redisCreateSSLContext(). + */ + +int redisInitiateSSLWithContext(redisContext *c, redisSSLContext *redis_ssl_ctx); + +/** + * Initiate SSL/TLS negotiation on a provided OpenSSL SSL object. */ int redisInitiateSSL(redisContext *c, struct ssl_st *ssl); +#ifdef __cplusplus +} +#endif + #endif /* __HIREDIS_SSL_H */ diff --git a/deps/hiredis/net.c b/deps/hiredis/net.c index e5f40b0a4..88f9aff25 100644 --- a/deps/hiredis/net.c +++ b/deps/hiredis/net.c @@ -57,8 +57,8 @@ void redisNetClose(redisContext *c) { } } -int redisNetRead(redisContext *c, char *buf, size_t bufcap) { - int nread = recv(c->fd, buf, bufcap, 0); +ssize_t redisNetRead(redisContext *c, char *buf, size_t bufcap) { + ssize_t nread = recv(c->fd, buf, bufcap, 0); if (nread == -1) { if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { /* Try again later */ @@ -79,8 +79,8 @@ int redisNetRead(redisContext *c, char *buf, size_t bufcap) { } } -int redisNetWrite(redisContext *c) { - int nwritten = send(c->fd, c->obuf, sdslen(c->obuf), 0); +ssize_t redisNetWrite(redisContext *c) { + ssize_t nwritten = send(c->fd, c->obuf, hi_sdslen(c->obuf), 0); if (nwritten < 0) { if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { /* Try again later */ @@ -203,7 +203,7 @@ int redisKeepAlive(redisContext *c, int interval) { return REDIS_OK; } -static int redisSetTcpNoDelay(redisContext *c) { +int redisSetTcpNoDelay(redisContext *c) { int yes = 1; if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)"); @@ -217,7 +217,7 @@ static int redisSetTcpNoDelay(redisContext *c) { static int redisContextTimeoutMsec(redisContext *c, long *result) { - const struct timeval *timeout = c->timeout; + const struct timeval *timeout = c->connect_timeout; long msec = -1; /* Only use timeout when not NULL. */ @@ -316,11 +316,7 @@ int redisCheckSocketError(redisContext *c) { int redisContextSetTimeout(redisContext *c, const struct timeval tv) { const void *to_ptr = &tv; size_t to_sz = sizeof(tv); -#ifdef _WIN32 - DWORD timeout_msec = tv.tv_sec * 1000 + tv.tv_usec / 1000; - to_ptr = &timeout_msec; - to_sz = sizeof(timeout_msec); -#endif + if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)"); return REDIS_ERR; @@ -332,6 +328,38 @@ int redisContextSetTimeout(redisContext *c, const struct timeval tv) { return REDIS_OK; } +int redisContextUpdateConnectTimeout(redisContext *c, const struct timeval *timeout) { + /* Same timeval struct, short circuit */ + if (c->connect_timeout == timeout) + return REDIS_OK; + + /* Allocate context timeval if we need to */ + if (c->connect_timeout == NULL) { + c->connect_timeout = hi_malloc(sizeof(*c->connect_timeout)); + if (c->connect_timeout == NULL) + return REDIS_ERR; + } + + memcpy(c->connect_timeout, timeout, sizeof(*c->connect_timeout)); + return REDIS_OK; +} + +int redisContextUpdateCommandTimeout(redisContext *c, const struct timeval *timeout) { + /* Same timeval struct, short circuit */ + if (c->command_timeout == timeout) + return REDIS_OK; + + /* Allocate context timeval if we need to */ + if (c->command_timeout == NULL) { + c->command_timeout = hi_malloc(sizeof(*c->command_timeout)); + if (c->command_timeout == NULL) + return REDIS_ERR; + } + + memcpy(c->command_timeout, timeout, sizeof(*c->command_timeout)); + return REDIS_OK; +} + static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout, const char *source_addr) { @@ -356,21 +384,19 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, * This is a bit ugly, but atleast it works and doesn't leak memory. **/ if (c->tcp.host != addr) { - free(c->tcp.host); + hi_free(c->tcp.host); - c->tcp.host = strdup(addr); + c->tcp.host = hi_strdup(addr); + if (c->tcp.host == NULL) + goto oom; } if (timeout) { - if (c->timeout != timeout) { - if (c->timeout == NULL) - c->timeout = malloc(sizeof(struct timeval)); - - memcpy(c->timeout, timeout, sizeof(struct timeval)); - } + if (redisContextUpdateConnectTimeout(c, timeout) == REDIS_ERR) + goto oom; } else { - free(c->timeout); - c->timeout = NULL; + hi_free(c->connect_timeout); + c->connect_timeout = NULL; } if (redisContextTimeoutMsec(c, &timeout_msec) != REDIS_OK) { @@ -379,11 +405,11 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, } if (source_addr == NULL) { - free(c->tcp.source_addr); + hi_free(c->tcp.source_addr); c->tcp.source_addr = NULL; } else if (c->tcp.source_addr != source_addr) { - free(c->tcp.source_addr); - c->tcp.source_addr = strdup(source_addr); + hi_free(c->tcp.source_addr); + c->tcp.source_addr = hi_strdup(source_addr); } snprintf(_port, 6, "%d", port); @@ -446,8 +472,11 @@ addrretry: } /* For repeat connection */ - free(c->saddr); - c->saddr = malloc(p->ai_addrlen); + hi_free(c->saddr); + c->saddr = hi_malloc(p->ai_addrlen); + if (c->saddr == NULL) + goto oom; + memcpy(c->saddr, p->ai_addr, p->ai_addrlen); c->addrlen = p->ai_addrlen; @@ -474,12 +503,12 @@ addrretry: wait_for_ready: if (redisContextWaitReady(c,timeout_msec) != REDIS_OK) goto error; + if (redisSetTcpNoDelay(c) != REDIS_OK) + goto error; } } if (blocking && redisSetBlocking(c,1) != REDIS_OK) goto error; - if (redisSetTcpNoDelay(c) != REDIS_OK) - goto error; c->flags |= REDIS_CONNECTED; rv = REDIS_OK; @@ -492,6 +521,8 @@ addrretry: goto error; } +oom: + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); error: rv = REDIS_ERR; end: @@ -525,25 +556,32 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time return REDIS_ERR; c->connection_type = REDIS_CONN_UNIX; - if (c->unix_sock.path != path) - c->unix_sock.path = strdup(path); + if (c->unix_sock.path != path) { + hi_free(c->unix_sock.path); + + c->unix_sock.path = hi_strdup(path); + if (c->unix_sock.path == NULL) + goto oom; + } if (timeout) { - if (c->timeout != timeout) { - if (c->timeout == NULL) - c->timeout = malloc(sizeof(struct timeval)); - - memcpy(c->timeout, timeout, sizeof(struct timeval)); - } + if (redisContextUpdateConnectTimeout(c, timeout) == REDIS_ERR) + goto oom; } else { - free(c->timeout); - c->timeout = NULL; + hi_free(c->connect_timeout); + c->connect_timeout = NULL; } if (redisContextTimeoutMsec(c,&timeout_msec) != REDIS_OK) return REDIS_ERR; - sa = (struct sockaddr_un*)(c->saddr = malloc(sizeof(struct sockaddr_un))); + /* Don't leak sockaddr if we're reconnecting */ + if (c->saddr) hi_free(c->saddr); + + sa = (struct sockaddr_un*)(c->saddr = hi_malloc(sizeof(struct sockaddr_un))); + if (sa == NULL) + goto oom; + c->addrlen = sizeof(struct sockaddr_un); sa->sun_family = AF_UNIX; strncpy(sa->sun_path, path, sizeof(sa->sun_path) - 1); @@ -568,4 +606,7 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time errno = EPROTONOSUPPORT; return REDIS_ERR; #endif /* _WIN32 */ +oom: + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; } diff --git a/deps/hiredis/net.h b/deps/hiredis/net.h index a4393c06b..9f43283a5 100644 --- a/deps/hiredis/net.h +++ b/deps/hiredis/net.h @@ -38,8 +38,8 @@ #include "hiredis.h" void redisNetClose(redisContext *c); -int redisNetRead(redisContext *c, char *buf, size_t bufcap); -int redisNetWrite(redisContext *c); +ssize_t redisNetRead(redisContext *c, char *buf, size_t bufcap); +ssize_t redisNetWrite(redisContext *c); int redisCheckSocketError(redisContext *c); int redisContextSetTimeout(redisContext *c, const struct timeval tv); @@ -51,4 +51,6 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time int redisKeepAlive(redisContext *c, int interval); int redisCheckConnectDone(redisContext *c, int *completed); +int redisSetTcpNoDelay(redisContext *c); + #endif diff --git a/deps/hiredis/read.c b/deps/hiredis/read.c index b9853ea9a..682b9a6b9 100644 --- a/deps/hiredis/read.c +++ b/deps/hiredis/read.c @@ -42,10 +42,14 @@ #include #include +#include "alloc.h" #include "read.h" #include "sds.h" #include "win32.h" +/* Initial size of our nested reply stack and how much we grow it when needd */ +#define REDIS_READER_STACK_SIZE 9 + static void __redisReaderSetError(redisReader *r, int type, const char *str) { size_t len; @@ -55,7 +59,7 @@ static void __redisReaderSetError(redisReader *r, int type, const char *str) { } /* Clear input buffer on errors. */ - sdsfree(r->buf); + hi_sdsfree(r->buf); r->buf = NULL; r->pos = r->len = 0; @@ -243,11 +247,12 @@ static void moveToNextTask(redisReader *r) { return; } - cur = &(r->rstack[r->ridx]); - prv = &(r->rstack[r->ridx-1]); + cur = r->task[r->ridx]; + prv = r->task[r->ridx-1]; assert(prv->type == REDIS_REPLY_ARRAY || prv->type == REDIS_REPLY_MAP || - prv->type == REDIS_REPLY_SET); + prv->type == REDIS_REPLY_SET || + prv->type == REDIS_REPLY_PUSH); if (cur->idx == prv->elements-1) { r->ridx--; } else { @@ -262,7 +267,7 @@ static void moveToNextTask(redisReader *r) { } static int processLineItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); + redisReadTask *cur = r->task[r->ridx]; void *obj; char *p; int len; @@ -297,7 +302,7 @@ static int processLineItem(redisReader *r) { if (strcasecmp(buf,",inf") == 0) { d = INFINITY; /* Positive infinite. */ } else if (strcasecmp(buf,",-inf") == 0) { - d = -INFINITY; /* Nevative infinite. */ + d = -INFINITY; /* Negative infinite. */ } else { d = strtod((char*)buf,&eptr); if (buf[0] == '\0' || eptr[0] != '\0' || isnan(d)) { @@ -344,7 +349,7 @@ static int processLineItem(redisReader *r) { } static int processBulkItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); + redisReadTask *cur = r->task[r->ridx]; void *obj = NULL; char *p, *s; long long len; @@ -415,19 +420,43 @@ static int processBulkItem(redisReader *r) { return REDIS_ERR; } +static int redisReaderGrow(redisReader *r) { + redisReadTask **aux; + int newlen; + + /* Grow our stack size */ + newlen = r->tasks + REDIS_READER_STACK_SIZE; + aux = hi_realloc(r->task, sizeof(*r->task) * newlen); + if (aux == NULL) + goto oom; + + r->task = aux; + + /* Allocate new tasks */ + for (; r->tasks < newlen; r->tasks++) { + r->task[r->tasks] = hi_calloc(1, sizeof(**r->task)); + if (r->task[r->tasks] == NULL) + goto oom; + } + + return REDIS_OK; +oom: + __redisReaderSetErrorOOM(r); + return REDIS_ERR; +} + /* Process the array, map and set types. */ static int processAggregateItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); + redisReadTask *cur = r->task[r->ridx]; void *obj; char *p; long long elements; int root = 0, len; /* Set error for nested multi bulks with depth > 7 */ - if (r->ridx == 8) { - __redisReaderSetError(r,REDIS_ERR_PROTOCOL, - "No support for nested multi bulk replies with depth > 7"); - return REDIS_ERR; + if (r->ridx == r->tasks - 1) { + if (redisReaderGrow(r) == REDIS_ERR) + return REDIS_ERR; } if ((p = readLine(r,&len)) != NULL) { @@ -439,7 +468,9 @@ static int processAggregateItem(redisReader *r) { root = (r->ridx == 0); - if (elements < -1 || (LLONG_MAX > SIZE_MAX && elements > SIZE_MAX)) { + if (elements < -1 || (LLONG_MAX > SIZE_MAX && elements > SIZE_MAX) || + (r->maxelements > 0 && elements > r->maxelements)) + { __redisReaderSetError(r,REDIS_ERR_PROTOCOL, "Multi-bulk length out of range"); return REDIS_ERR; @@ -475,12 +506,12 @@ static int processAggregateItem(redisReader *r) { cur->elements = elements; cur->obj = obj; r->ridx++; - r->rstack[r->ridx].type = -1; - r->rstack[r->ridx].elements = -1; - r->rstack[r->ridx].idx = 0; - r->rstack[r->ridx].obj = NULL; - r->rstack[r->ridx].parent = cur; - r->rstack[r->ridx].privdata = r->privdata; + r->task[r->ridx]->type = -1; + r->task[r->ridx]->elements = -1; + r->task[r->ridx]->idx = 0; + r->task[r->ridx]->obj = NULL; + r->task[r->ridx]->parent = cur; + r->task[r->ridx]->privdata = r->privdata; } else { moveToNextTask(r); } @@ -495,7 +526,7 @@ static int processAggregateItem(redisReader *r) { } static int processItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); + redisReadTask *cur = r->task[r->ridx]; char *p; /* check if we need to read type */ @@ -535,6 +566,9 @@ static int processItem(redisReader *r) { case '=': cur->type = REDIS_REPLY_VERB; break; + case '>': + cur->type = REDIS_REPLY_PUSH; + break; default: __redisReaderSetErrorProtocolByte(r,*p); return REDIS_ERR; @@ -560,6 +594,7 @@ static int processItem(redisReader *r) { case REDIS_REPLY_ARRAY: case REDIS_REPLY_MAP: case REDIS_REPLY_SET: + case REDIS_REPLY_PUSH: return processAggregateItem(r); default: assert(NULL); @@ -570,33 +605,57 @@ static int processItem(redisReader *r) { redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) { redisReader *r; - r = calloc(1,sizeof(redisReader)); + r = hi_calloc(1,sizeof(redisReader)); if (r == NULL) return NULL; - r->fn = fn; - r->buf = sdsempty(); - r->maxbuf = REDIS_READER_MAX_BUF; - if (r->buf == NULL) { - free(r); - return NULL; + r->buf = hi_sdsempty(); + if (r->buf == NULL) + goto oom; + + r->task = hi_calloc(REDIS_READER_STACK_SIZE, sizeof(*r->task)); + if (r->task == NULL) + goto oom; + + for (; r->tasks < REDIS_READER_STACK_SIZE; r->tasks++) { + r->task[r->tasks] = hi_calloc(1, sizeof(**r->task)); + if (r->task[r->tasks] == NULL) + goto oom; } + r->fn = fn; + r->maxbuf = REDIS_READER_MAX_BUF; + r->maxelements = REDIS_READER_MAX_ARRAY_ELEMENTS; r->ridx = -1; + return r; +oom: + redisReaderFree(r); + return NULL; } void redisReaderFree(redisReader *r) { if (r == NULL) return; + if (r->reply != NULL && r->fn && r->fn->freeObject) r->fn->freeObject(r->reply); - sdsfree(r->buf); - free(r); + + if (r->task) { + /* We know r->task[i] is allocated if i < r->tasks */ + for (int i = 0; i < r->tasks; i++) { + hi_free(r->task[i]); + } + + hi_free(r->task); + } + + hi_sdsfree(r->buf); + hi_free(r); } int redisReaderFeed(redisReader *r, const char *buf, size_t len) { - sds newbuf; + hisds newbuf; /* Return early when this reader is in an erroneous state. */ if (r->err) @@ -605,26 +664,25 @@ int redisReaderFeed(redisReader *r, const char *buf, size_t len) { /* Copy the provided buffer. */ if (buf != NULL && len >= 1) { /* Destroy internal buffer when it is empty and is quite large. */ - if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) { - sdsfree(r->buf); - r->buf = sdsempty(); + if (r->len == 0 && r->maxbuf != 0 && hi_sdsavail(r->buf) > r->maxbuf) { + hi_sdsfree(r->buf); + r->buf = hi_sdsempty(); + if (r->buf == 0) goto oom; + r->pos = 0; - - /* r->buf should not be NULL since we just free'd a larger one. */ - assert(r->buf != NULL); } - newbuf = sdscatlen(r->buf,buf,len); - if (newbuf == NULL) { - __redisReaderSetErrorOOM(r); - return REDIS_ERR; - } + newbuf = hi_sdscatlen(r->buf,buf,len); + if (newbuf == NULL) goto oom; r->buf = newbuf; - r->len = sdslen(r->buf); + r->len = hi_sdslen(r->buf); } return REDIS_OK; +oom: + __redisReaderSetErrorOOM(r); + return REDIS_ERR; } int redisReaderGetReply(redisReader *r, void **reply) { @@ -642,12 +700,12 @@ int redisReaderGetReply(redisReader *r, void **reply) { /* Set first item to process when the stack is empty. */ if (r->ridx == -1) { - r->rstack[0].type = -1; - r->rstack[0].elements = -1; - r->rstack[0].idx = -1; - r->rstack[0].obj = NULL; - r->rstack[0].parent = NULL; - r->rstack[0].privdata = r->privdata; + r->task[0]->type = -1; + r->task[0]->elements = -1; + r->task[0]->idx = -1; + r->task[0]->obj = NULL; + r->task[0]->parent = NULL; + r->task[0]->privdata = r->privdata; r->ridx = 0; } @@ -663,9 +721,9 @@ int redisReaderGetReply(redisReader *r, void **reply) { /* Discard part of the buffer when we've consumed at least 1k, to avoid * doing unnecessary calls to memmove() in sds.c. */ if (r->pos >= 1024) { - sdsrange(r->buf,r->pos,-1); + if (hi_sdsrange(r->buf,r->pos,-1) < 0) return REDIS_ERR; r->pos = 0; - r->len = sdslen(r->buf); + r->len = hi_sdslen(r->buf); } /* Emit a reply when there is one. */ diff --git a/deps/hiredis/read.h b/deps/hiredis/read.h index 58105312a..2d74d77a5 100644 --- a/deps/hiredis/read.h +++ b/deps/hiredis/read.h @@ -63,7 +63,11 @@ #define REDIS_REPLY_BIGNUM 13 #define REDIS_REPLY_VERB 14 -#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */ +/* Default max unused reader buffer. */ +#define REDIS_READER_MAX_BUF (1024*16) + +/* Default multi-bulk element limit */ +#define REDIS_READER_MAX_ARRAY_ELEMENTS ((1LL<<32) - 1) #ifdef __cplusplus extern "C" { @@ -71,7 +75,7 @@ extern "C" { typedef struct redisReadTask { int type; - int elements; /* number of elements in multibulk container */ + long long elements; /* number of elements in multibulk container */ int idx; /* index in parent (array) object */ void *obj; /* holds user-generated value for a read task */ struct redisReadTask *parent; /* parent task */ @@ -96,8 +100,11 @@ typedef struct redisReader { size_t pos; /* Buffer cursor */ size_t len; /* Buffer length */ size_t maxbuf; /* Max length of unused buffer */ + long long maxelements; /* Max multi-bulk elements */ + + redisReadTask **task; + int tasks; - redisReadTask rstack[9]; int ridx; /* Index of current read task */ void *reply; /* Temporary reply pointer */ diff --git a/deps/hiredis/sds.c b/deps/hiredis/sds.c index 6cf75841c..675e7649f 100644 --- a/deps/hiredis/sds.c +++ b/deps/hiredis/sds.c @@ -30,98 +30,100 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#include "fmacros.h" #include #include #include #include #include +#include #include "sds.h" #include "sdsalloc.h" -static inline int sdsHdrSize(char type) { - switch(type&SDS_TYPE_MASK) { - case SDS_TYPE_5: - return sizeof(struct sdshdr5); - case SDS_TYPE_8: - return sizeof(struct sdshdr8); - case SDS_TYPE_16: - return sizeof(struct sdshdr16); - case SDS_TYPE_32: - return sizeof(struct sdshdr32); - case SDS_TYPE_64: - return sizeof(struct sdshdr64); +static inline int hi_sdsHdrSize(char type) { + switch(type&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: + return sizeof(struct hisdshdr5); + case HI_SDS_TYPE_8: + return sizeof(struct hisdshdr8); + case HI_SDS_TYPE_16: + return sizeof(struct hisdshdr16); + case HI_SDS_TYPE_32: + return sizeof(struct hisdshdr32); + case HI_SDS_TYPE_64: + return sizeof(struct hisdshdr64); } return 0; } -static inline char sdsReqType(size_t string_size) { +static inline char hi_sdsReqType(size_t string_size) { if (string_size < 32) - return SDS_TYPE_5; + return HI_SDS_TYPE_5; if (string_size < 0xff) - return SDS_TYPE_8; + return HI_SDS_TYPE_8; if (string_size < 0xffff) - return SDS_TYPE_16; + return HI_SDS_TYPE_16; if (string_size < 0xffffffff) - return SDS_TYPE_32; - return SDS_TYPE_64; + return HI_SDS_TYPE_32; + return HI_SDS_TYPE_64; } -/* Create a new sds string with the content specified by the 'init' pointer +/* Create a new hisds string with the content specified by the 'init' pointer * and 'initlen'. * If NULL is used for 'init' the string is initialized with zero bytes. * - * The string is always null-termined (all the sds strings are, always) so - * even if you create an sds string with: + * The string is always null-termined (all the hisds strings are, always) so + * even if you create an hisds string with: * - * mystring = sdsnewlen("abc",3); + * mystring = hi_sdsnewlen("abc",3); * * You can print the string with printf() as there is an implicit \0 at the * end of the string. However the string is binary safe and can contain - * \0 characters in the middle, as the length is stored in the sds header. */ -sds sdsnewlen(const void *init, size_t initlen) { + * \0 characters in the middle, as the length is stored in the hisds header. */ +hisds hi_sdsnewlen(const void *init, size_t initlen) { void *sh; - sds s; - char type = sdsReqType(initlen); + hisds s; + char type = hi_sdsReqType(initlen); /* Empty strings are usually created in order to append. Use type 8 * since type 5 is not good at this. */ - if (type == SDS_TYPE_5 && initlen == 0) type = SDS_TYPE_8; - int hdrlen = sdsHdrSize(type); + if (type == HI_SDS_TYPE_5 && initlen == 0) type = HI_SDS_TYPE_8; + int hdrlen = hi_sdsHdrSize(type); unsigned char *fp; /* flags pointer. */ - sh = s_malloc(hdrlen+initlen+1); + sh = hi_s_malloc(hdrlen+initlen+1); if (sh == NULL) return NULL; if (!init) memset(sh, 0, hdrlen+initlen+1); s = (char*)sh+hdrlen; fp = ((unsigned char*)s)-1; switch(type) { - case SDS_TYPE_5: { - *fp = type | (initlen << SDS_TYPE_BITS); + case HI_SDS_TYPE_5: { + *fp = type | (initlen << HI_SDS_TYPE_BITS); break; } - case SDS_TYPE_8: { - SDS_HDR_VAR(8,s); + case HI_SDS_TYPE_8: { + HI_SDS_HDR_VAR(8,s); sh->len = initlen; sh->alloc = initlen; *fp = type; break; } - case SDS_TYPE_16: { - SDS_HDR_VAR(16,s); + case HI_SDS_TYPE_16: { + HI_SDS_HDR_VAR(16,s); sh->len = initlen; sh->alloc = initlen; *fp = type; break; } - case SDS_TYPE_32: { - SDS_HDR_VAR(32,s); + case HI_SDS_TYPE_32: { + HI_SDS_HDR_VAR(32,s); sh->len = initlen; sh->alloc = initlen; *fp = type; break; } - case SDS_TYPE_64: { - SDS_HDR_VAR(64,s); + case HI_SDS_TYPE_64: { + HI_SDS_HDR_VAR(64,s); sh->len = initlen; sh->alloc = initlen; *fp = type; @@ -134,167 +136,164 @@ sds sdsnewlen(const void *init, size_t initlen) { return s; } -/* Create an empty (zero length) sds string. Even in this case the string +/* Create an empty (zero length) hisds string. Even in this case the string * always has an implicit null term. */ -sds sdsempty(void) { - return sdsnewlen("",0); +hisds hi_sdsempty(void) { + return hi_sdsnewlen("",0); } -/* Create a new sds string starting from a null terminated C string. */ -sds sdsnew(const char *init) { +/* Create a new hisds string starting from a null terminated C string. */ +hisds hi_sdsnew(const char *init) { size_t initlen = (init == NULL) ? 0 : strlen(init); - return sdsnewlen(init, initlen); + return hi_sdsnewlen(init, initlen); } -/* Duplicate an sds string. */ -sds sdsdup(const sds s) { - return sdsnewlen(s, sdslen(s)); +/* Duplicate an hisds string. */ +hisds hi_sdsdup(const hisds s) { + return hi_sdsnewlen(s, hi_sdslen(s)); } -/* Free an sds string. No operation is performed if 's' is NULL. */ -void sdsfree(sds s) { +/* Free an hisds string. No operation is performed if 's' is NULL. */ +void hi_sdsfree(hisds s) { if (s == NULL) return; - s_free((char*)s-sdsHdrSize(s[-1])); + hi_s_free((char*)s-hi_sdsHdrSize(s[-1])); } -/* Set the sds string length to the length as obtained with strlen(), so +/* Set the hisds string length to the length as obtained with strlen(), so * considering as content only up to the first null term character. * - * This function is useful when the sds string is hacked manually in some + * This function is useful when the hisds string is hacked manually in some * way, like in the following example: * - * s = sdsnew("foobar"); + * s = hi_sdsnew("foobar"); * s[2] = '\0'; - * sdsupdatelen(s); - * printf("%d\n", sdslen(s)); + * hi_sdsupdatelen(s); + * printf("%d\n", hi_sdslen(s)); * - * The output will be "2", but if we comment out the call to sdsupdatelen() + * The output will be "2", but if we comment out the call to hi_sdsupdatelen() * the output will be "6" as the string was modified but the logical length * remains 6 bytes. */ -void sdsupdatelen(sds s) { +void hi_sdsupdatelen(hisds s) { int reallen = strlen(s); - sdssetlen(s, reallen); + hi_sdssetlen(s, reallen); } -/* Modify an sds string in-place to make it empty (zero length). +/* Modify an hisds string in-place to make it empty (zero length). * However all the existing buffer is not discarded but set as free space * so that next append operations will not require allocations up to the * number of bytes previously available. */ -void sdsclear(sds s) { - sdssetlen(s, 0); +void hi_sdsclear(hisds s) { + hi_sdssetlen(s, 0); s[0] = '\0'; } -/* Enlarge the free space at the end of the sds string so that the caller +/* Enlarge the free space at the end of the hisds string so that the caller * is sure that after calling this function can overwrite up to addlen * bytes after the end of the string, plus one more byte for nul term. * - * Note: this does not change the *length* of the sds string as returned - * by sdslen(), but only the free buffer space we have. */ -sds sdsMakeRoomFor(sds s, size_t addlen) { + * Note: this does not change the *length* of the hisds string as returned + * by hi_sdslen(), but only the free buffer space we have. */ +hisds hi_sdsMakeRoomFor(hisds s, size_t addlen) { void *sh, *newsh; - size_t avail = sdsavail(s); + size_t avail = hi_sdsavail(s); size_t len, newlen; - char type, oldtype = s[-1] & SDS_TYPE_MASK; + char type, oldtype = s[-1] & HI_SDS_TYPE_MASK; int hdrlen; /* Return ASAP if there is enough space left. */ if (avail >= addlen) return s; - len = sdslen(s); - sh = (char*)s-sdsHdrSize(oldtype); + len = hi_sdslen(s); + sh = (char*)s-hi_sdsHdrSize(oldtype); newlen = (len+addlen); - if (newlen < SDS_MAX_PREALLOC) + if (newlen < HI_SDS_MAX_PREALLOC) newlen *= 2; else - newlen += SDS_MAX_PREALLOC; + newlen += HI_SDS_MAX_PREALLOC; - type = sdsReqType(newlen); + type = hi_sdsReqType(newlen); /* Don't use type 5: the user is appending to the string and type 5 is - * not able to remember empty space, so sdsMakeRoomFor() must be called + * not able to remember empty space, so hi_sdsMakeRoomFor() must be called * at every appending operation. */ - if (type == SDS_TYPE_5) type = SDS_TYPE_8; + if (type == HI_SDS_TYPE_5) type = HI_SDS_TYPE_8; - hdrlen = sdsHdrSize(type); + hdrlen = hi_sdsHdrSize(type); if (oldtype==type) { - newsh = s_realloc(sh, hdrlen+newlen+1); - if (newsh == NULL) { - s_free(sh); - return NULL; - } + newsh = hi_s_realloc(sh, hdrlen+newlen+1); + if (newsh == NULL) return NULL; s = (char*)newsh+hdrlen; } else { /* Since the header size changes, need to move the string forward, * and can't use realloc */ - newsh = s_malloc(hdrlen+newlen+1); + newsh = hi_s_malloc(hdrlen+newlen+1); if (newsh == NULL) return NULL; memcpy((char*)newsh+hdrlen, s, len+1); - s_free(sh); + hi_s_free(sh); s = (char*)newsh+hdrlen; s[-1] = type; - sdssetlen(s, len); + hi_sdssetlen(s, len); } - sdssetalloc(s, newlen); + hi_sdssetalloc(s, newlen); return s; } -/* Reallocate the sds string so that it has no free space at the end. The +/* Reallocate the hisds string so that it has no free space at the end. The * contained string remains not altered, but next concatenation operations * will require a reallocation. * - * After the call, the passed sds string is no longer valid and all the + * After the call, the passed hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdsRemoveFreeSpace(sds s) { +hisds hi_sdsRemoveFreeSpace(hisds s) { void *sh, *newsh; - char type, oldtype = s[-1] & SDS_TYPE_MASK; + char type, oldtype = s[-1] & HI_SDS_TYPE_MASK; int hdrlen; - size_t len = sdslen(s); - sh = (char*)s-sdsHdrSize(oldtype); + size_t len = hi_sdslen(s); + sh = (char*)s-hi_sdsHdrSize(oldtype); - type = sdsReqType(len); - hdrlen = sdsHdrSize(type); + type = hi_sdsReqType(len); + hdrlen = hi_sdsHdrSize(type); if (oldtype==type) { - newsh = s_realloc(sh, hdrlen+len+1); + newsh = hi_s_realloc(sh, hdrlen+len+1); if (newsh == NULL) return NULL; s = (char*)newsh+hdrlen; } else { - newsh = s_malloc(hdrlen+len+1); + newsh = hi_s_malloc(hdrlen+len+1); if (newsh == NULL) return NULL; memcpy((char*)newsh+hdrlen, s, len+1); - s_free(sh); + hi_s_free(sh); s = (char*)newsh+hdrlen; s[-1] = type; - sdssetlen(s, len); + hi_sdssetlen(s, len); } - sdssetalloc(s, len); + hi_sdssetalloc(s, len); return s; } -/* Return the total size of the allocation of the specifed sds string, +/* Return the total size of the allocation of the specifed hisds string, * including: - * 1) The sds header before the pointer. + * 1) The hisds header before the pointer. * 2) The string. * 3) The free buffer at the end if any. * 4) The implicit null term. */ -size_t sdsAllocSize(sds s) { - size_t alloc = sdsalloc(s); - return sdsHdrSize(s[-1])+alloc+1; +size_t hi_sdsAllocSize(hisds s) { + size_t alloc = hi_sdsalloc(s); + return hi_sdsHdrSize(s[-1])+alloc+1; } /* Return the pointer of the actual SDS allocation (normally SDS strings * are referenced by the start of the string buffer). */ -void *sdsAllocPtr(sds s) { - return (void*) (s-sdsHdrSize(s[-1])); +void *hi_sdsAllocPtr(hisds s) { + return (void*) (s-hi_sdsHdrSize(s[-1])); } -/* Increment the sds length and decrements the left free space at the +/* Increment the hisds length and decrements the left free space at the * end of the string according to 'incr'. Also set the null term * in the new end of the string. * * This function is used in order to fix the string length after the - * user calls sdsMakeRoomFor(), writes something after the end of + * user calls hi_sdsMakeRoomFor(), writes something after the end of * the current string, and finally needs to set the new length. * * Note: it is possible to use a negative increment in order to @@ -302,48 +301,48 @@ void *sdsAllocPtr(sds s) { * * Usage example: * - * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the + * Using hi_sdsIncrLen() and hi_sdsMakeRoomFor() it is possible to mount the * following schema, to cat bytes coming from the kernel to the end of an - * sds string without copying into an intermediate buffer: + * hisds string without copying into an intermediate buffer: * - * oldlen = sdslen(s); - * s = sdsMakeRoomFor(s, BUFFER_SIZE); + * oldlen = hi_hi_sdslen(s); + * s = hi_sdsMakeRoomFor(s, BUFFER_SIZE); * nread = read(fd, s+oldlen, BUFFER_SIZE); * ... check for nread <= 0 and handle it ... - * sdsIncrLen(s, nread); + * hi_sdsIncrLen(s, nread); */ -void sdsIncrLen(sds s, int incr) { +void hi_sdsIncrLen(hisds s, int incr) { unsigned char flags = s[-1]; size_t len; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: { + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - unsigned char oldlen = SDS_TYPE_5_LEN(flags); + unsigned char oldlen = HI_SDS_TYPE_5_LEN(flags); assert((incr > 0 && oldlen+incr < 32) || (incr < 0 && oldlen >= (unsigned int)(-incr))); - *fp = SDS_TYPE_5 | ((oldlen+incr) << SDS_TYPE_BITS); + *fp = HI_SDS_TYPE_5 | ((oldlen+incr) << HI_SDS_TYPE_BITS); len = oldlen+incr; break; } - case SDS_TYPE_8: { - SDS_HDR_VAR(8,s); + case HI_SDS_TYPE_8: { + HI_SDS_HDR_VAR(8,s); assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); len = (sh->len += incr); break; } - case SDS_TYPE_16: { - SDS_HDR_VAR(16,s); + case HI_SDS_TYPE_16: { + HI_SDS_HDR_VAR(16,s); assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); len = (sh->len += incr); break; } - case SDS_TYPE_32: { - SDS_HDR_VAR(32,s); + case HI_SDS_TYPE_32: { + HI_SDS_HDR_VAR(32,s); assert((incr >= 0 && sh->alloc-sh->len >= (unsigned int)incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); len = (sh->len += incr); break; } - case SDS_TYPE_64: { - SDS_HDR_VAR(64,s); + case HI_SDS_TYPE_64: { + HI_SDS_HDR_VAR(64,s); assert((incr >= 0 && sh->alloc-sh->len >= (uint64_t)incr) || (incr < 0 && sh->len >= (uint64_t)(-incr))); len = (sh->len += incr); break; @@ -353,83 +352,83 @@ void sdsIncrLen(sds s, int incr) { s[len] = '\0'; } -/* Grow the sds to have the specified length. Bytes that were not part of - * the original length of the sds will be set to zero. +/* Grow the hisds to have the specified length. Bytes that were not part of + * the original length of the hisds will be set to zero. * * if the specified length is smaller than the current length, no operation * is performed. */ -sds sdsgrowzero(sds s, size_t len) { - size_t curlen = sdslen(s); +hisds hi_sdsgrowzero(hisds s, size_t len) { + size_t curlen = hi_sdslen(s); if (len <= curlen) return s; - s = sdsMakeRoomFor(s,len-curlen); + s = hi_sdsMakeRoomFor(s,len-curlen); if (s == NULL) return NULL; /* Make sure added region doesn't contain garbage */ memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */ - sdssetlen(s, len); + hi_sdssetlen(s, len); return s; } /* Append the specified binary-safe string pointed by 't' of 'len' bytes to the - * end of the specified sds string 's'. + * end of the specified hisds string 's'. * - * After the call, the passed sds string is no longer valid and all the + * After the call, the passed hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdscatlen(sds s, const void *t, size_t len) { - size_t curlen = sdslen(s); +hisds hi_sdscatlen(hisds s, const void *t, size_t len) { + size_t curlen = hi_sdslen(s); - s = sdsMakeRoomFor(s,len); + s = hi_sdsMakeRoomFor(s,len); if (s == NULL) return NULL; memcpy(s+curlen, t, len); - sdssetlen(s, curlen+len); + hi_sdssetlen(s, curlen+len); s[curlen+len] = '\0'; return s; } -/* Append the specified null termianted C string to the sds string 's'. +/* Append the specified null termianted C string to the hisds string 's'. * - * After the call, the passed sds string is no longer valid and all the + * After the call, the passed hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdscat(sds s, const char *t) { - return sdscatlen(s, t, strlen(t)); +hisds hi_sdscat(hisds s, const char *t) { + return hi_sdscatlen(s, t, strlen(t)); } -/* Append the specified sds 't' to the existing sds 's'. +/* Append the specified hisds 't' to the existing hisds 's'. * - * After the call, the modified sds string is no longer valid and all the + * After the call, the modified hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdscatsds(sds s, const sds t) { - return sdscatlen(s, t, sdslen(t)); +hisds hi_sdscatsds(hisds s, const hisds t) { + return hi_sdscatlen(s, t, hi_sdslen(t)); } -/* Destructively modify the sds string 's' to hold the specified binary +/* Destructively modify the hisds string 's' to hold the specified binary * safe string pointed by 't' of length 'len' bytes. */ -sds sdscpylen(sds s, const char *t, size_t len) { - if (sdsalloc(s) < len) { - s = sdsMakeRoomFor(s,len-sdslen(s)); +hisds hi_sdscpylen(hisds s, const char *t, size_t len) { + if (hi_sdsalloc(s) < len) { + s = hi_sdsMakeRoomFor(s,len-hi_sdslen(s)); if (s == NULL) return NULL; } memcpy(s, t, len); s[len] = '\0'; - sdssetlen(s, len); + hi_sdssetlen(s, len); return s; } -/* Like sdscpylen() but 't' must be a null-termined string so that the length +/* Like hi_sdscpylen() but 't' must be a null-termined string so that the length * of the string is obtained with strlen(). */ -sds sdscpy(sds s, const char *t) { - return sdscpylen(s, t, strlen(t)); +hisds hi_sdscpy(hisds s, const char *t) { + return hi_sdscpylen(s, t, strlen(t)); } -/* Helper for sdscatlonglong() doing the actual number -> string +/* Helper for hi_sdscatlonglong() doing the actual number -> string * conversion. 's' must point to a string with room for at least - * SDS_LLSTR_SIZE bytes. + * HI_SDS_LLSTR_SIZE bytes. * * The function returns the length of the null-terminated string * representation stored at 's'. */ -#define SDS_LLSTR_SIZE 21 -int sdsll2str(char *s, long long value) { +#define HI_SDS_LLSTR_SIZE 21 +int hi_sdsll2str(char *s, long long value) { char *p, aux; unsigned long long v; size_t l; @@ -460,8 +459,8 @@ int sdsll2str(char *s, long long value) { return l; } -/* Identical sdsll2str(), but for unsigned long long type. */ -int sdsull2str(char *s, unsigned long long v) { +/* Identical hi_sdsll2str(), but for unsigned long long type. */ +int hi_sdsull2str(char *s, unsigned long long v) { char *p, aux; size_t l; @@ -489,19 +488,19 @@ int sdsull2str(char *s, unsigned long long v) { return l; } -/* Create an sds string from a long long value. It is much faster than: +/* Create an hisds string from a long long value. It is much faster than: * - * sdscatprintf(sdsempty(),"%lld\n", value); + * hi_sdscatprintf(hi_sdsempty(),"%lld\n", value); */ -sds sdsfromlonglong(long long value) { - char buf[SDS_LLSTR_SIZE]; - int len = sdsll2str(buf,value); +hisds hi_sdsfromlonglong(long long value) { + char buf[HI_SDS_LLSTR_SIZE]; + int len = hi_sdsll2str(buf,value); - return sdsnewlen(buf,len); + return hi_sdsnewlen(buf,len); } -/* Like sdscatprintf() but gets va_list instead of being variadic. */ -sds sdscatvprintf(sds s, const char *fmt, va_list ap) { +/* Like hi_sdscatprintf() but gets va_list instead of being variadic. */ +hisds hi_sdscatvprintf(hisds s, const char *fmt, va_list ap) { va_list cpy; char staticbuf[1024], *buf = staticbuf, *t; size_t buflen = strlen(fmt)*2; @@ -509,7 +508,7 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { /* We try to start using a static buffer for speed. * If not possible we revert to heap allocation. */ if (buflen > sizeof(staticbuf)) { - buf = s_malloc(buflen); + buf = hi_s_malloc(buflen); if (buf == NULL) return NULL; } else { buflen = sizeof(staticbuf); @@ -523,9 +522,9 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { vsnprintf(buf, buflen, fmt, cpy); va_end(cpy); if (buf[buflen-2] != '\0') { - if (buf != staticbuf) s_free(buf); + if (buf != staticbuf) hi_s_free(buf); buflen *= 2; - buf = s_malloc(buflen); + buf = hi_s_malloc(buflen); if (buf == NULL) return NULL; continue; } @@ -533,39 +532,39 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { } /* Finally concat the obtained string to the SDS string and return it. */ - t = sdscat(s, buf); - if (buf != staticbuf) s_free(buf); + t = hi_sdscat(s, buf); + if (buf != staticbuf) hi_s_free(buf); return t; } -/* Append to the sds string 's' a string obtained using printf-alike format +/* Append to the hisds string 's' a string obtained using printf-alike format * specifier. * - * After the call, the modified sds string is no longer valid and all the + * After the call, the modified hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. * * Example: * - * s = sdsnew("Sum is: "); - * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b). + * s = hi_sdsnew("Sum is: "); + * s = hi_sdscatprintf(s,"%d+%d = %d",a,b,a+b). * * Often you need to create a string from scratch with the printf-alike - * format. When this is the need, just use sdsempty() as the target string: + * format. When this is the need, just use hi_sdsempty() as the target string: * - * s = sdscatprintf(sdsempty(), "... your format ...", args); + * s = hi_sdscatprintf(hi_sdsempty(), "... your format ...", args); */ -sds sdscatprintf(sds s, const char *fmt, ...) { +hisds hi_sdscatprintf(hisds s, const char *fmt, ...) { va_list ap; char *t; va_start(ap, fmt); - t = sdscatvprintf(s,fmt,ap); + t = hi_sdscatvprintf(s,fmt,ap); va_end(ap); return t; } -/* This function is similar to sdscatprintf, but much faster as it does +/* This function is similar to hi_sdscatprintf, but much faster as it does * not rely on sprintf() family functions implemented by the libc that - * are often very slow. Moreover directly handling the sds string as + * are often very slow. Moreover directly handling the hisds string as * new data is concatenated provides a performance improvement. * * However this function only handles an incompatible subset of printf-alike @@ -579,13 +578,13 @@ sds sdscatprintf(sds s, const char *fmt, ...) { * %U - 64 bit unsigned integer (unsigned long long, uint64_t) * %% - Verbatim "%" character. */ -sds sdscatfmt(sds s, char const *fmt, ...) { +hisds hi_sdscatfmt(hisds s, char const *fmt, ...) { const char *f = fmt; int i; va_list ap; va_start(ap,fmt); - i = sdslen(s); /* Position of the next byte to write to dest str. */ + i = hi_sdslen(s); /* Position of the next byte to write to dest str. */ while(*f) { char next, *str; size_t l; @@ -593,8 +592,8 @@ sds sdscatfmt(sds s, char const *fmt, ...) { unsigned long long unum; /* Make sure there is always space for at least 1 char. */ - if (sdsavail(s)==0) { - s = sdsMakeRoomFor(s,1); + if (hi_sdsavail(s)==0) { + s = hi_sdsMakeRoomFor(s,1); if (s == NULL) goto fmt_error; } @@ -606,13 +605,13 @@ sds sdscatfmt(sds s, char const *fmt, ...) { case 's': case 'S': str = va_arg(ap,char*); - l = (next == 's') ? strlen(str) : sdslen(str); - if (sdsavail(s) < l) { - s = sdsMakeRoomFor(s,l); + l = (next == 's') ? strlen(str) : hi_sdslen(str); + if (hi_sdsavail(s) < l) { + s = hi_sdsMakeRoomFor(s,l); if (s == NULL) goto fmt_error; } memcpy(s+i,str,l); - sdsinclen(s,l); + hi_sdsinclen(s,l); i += l; break; case 'i': @@ -622,14 +621,14 @@ sds sdscatfmt(sds s, char const *fmt, ...) { else num = va_arg(ap,long long); { - char buf[SDS_LLSTR_SIZE]; - l = sdsll2str(buf,num); - if (sdsavail(s) < l) { - s = sdsMakeRoomFor(s,l); + char buf[HI_SDS_LLSTR_SIZE]; + l = hi_sdsll2str(buf,num); + if (hi_sdsavail(s) < l) { + s = hi_sdsMakeRoomFor(s,l); if (s == NULL) goto fmt_error; } memcpy(s+i,buf,l); - sdsinclen(s,l); + hi_sdsinclen(s,l); i += l; } break; @@ -640,26 +639,26 @@ sds sdscatfmt(sds s, char const *fmt, ...) { else unum = va_arg(ap,unsigned long long); { - char buf[SDS_LLSTR_SIZE]; - l = sdsull2str(buf,unum); - if (sdsavail(s) < l) { - s = sdsMakeRoomFor(s,l); + char buf[HI_SDS_LLSTR_SIZE]; + l = hi_sdsull2str(buf,unum); + if (hi_sdsavail(s) < l) { + s = hi_sdsMakeRoomFor(s,l); if (s == NULL) goto fmt_error; } memcpy(s+i,buf,l); - sdsinclen(s,l); + hi_sdsinclen(s,l); i += l; } break; default: /* Handle %% and generally %. */ s[i++] = next; - sdsinclen(s,1); + hi_sdsinclen(s,1); break; } break; default: s[i++] = *f; - sdsinclen(s,1); + hi_sdsinclen(s,1); break; } f++; @@ -678,29 +677,29 @@ fmt_error: /* Remove the part of the string from left and from right composed just of * contiguous characters found in 'cset', that is a null terminted C string. * - * After the call, the modified sds string is no longer valid and all the + * After the call, the modified hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. * * Example: * - * s = sdsnew("AA...AA.a.aa.aHelloWorld :::"); - * s = sdstrim(s,"Aa. :"); + * s = hi_sdsnew("AA...AA.a.aa.aHelloWorld :::"); + * s = hi_sdstrim(s,"Aa. :"); * printf("%s\n", s); * * Output will be just "Hello World". */ -sds sdstrim(sds s, const char *cset) { +hisds hi_sdstrim(hisds s, const char *cset) { char *start, *end, *sp, *ep; size_t len; sp = start = s; - ep = end = s+sdslen(s)-1; + ep = end = s+hi_sdslen(s)-1; while(sp <= end && strchr(cset, *sp)) sp++; while(ep > sp && strchr(cset, *ep)) ep--; len = (sp > ep) ? 0 : ((ep-sp)+1); if (s != sp) memmove(s, sp, len); s[len] = '\0'; - sdssetlen(s,len); + hi_sdssetlen(s,len); return s; } @@ -715,15 +714,20 @@ sds sdstrim(sds s, const char *cset) { * * The string is modified in-place. * + * Return value: + * -1 (error) if hi_sdslen(s) is larger than maximum positive ssize_t value. + * 0 on success. + * * Example: * - * s = sdsnew("Hello World"); - * sdsrange(s,1,-1); => "ello World" + * s = hi_sdsnew("Hello World"); + * hi_sdsrange(s,1,-1); => "ello World" */ -void sdsrange(sds s, int start, int end) { - size_t newlen, len = sdslen(s); +int hi_sdsrange(hisds s, ssize_t start, ssize_t end) { + size_t newlen, len = hi_sdslen(s); + if (len > SSIZE_MAX) return -1; - if (len == 0) return; + if (len == 0) return 0; if (start < 0) { start = len+start; if (start < 0) start = 0; @@ -734,9 +738,9 @@ void sdsrange(sds s, int start, int end) { } newlen = (start > end) ? 0 : (end-start)+1; if (newlen != 0) { - if (start >= (signed)len) { + if (start >= (ssize_t)len) { newlen = 0; - } else if (end >= (signed)len) { + } else if (end >= (ssize_t)len) { end = len-1; newlen = (start > end) ? 0 : (end-start)+1; } @@ -745,24 +749,25 @@ void sdsrange(sds s, int start, int end) { } if (start && newlen) memmove(s, s+start, newlen); s[newlen] = 0; - sdssetlen(s,newlen); + hi_sdssetlen(s,newlen); + return 0; } -/* Apply tolower() to every character of the sds string 's'. */ -void sdstolower(sds s) { - int len = sdslen(s), j; +/* Apply tolower() to every character of the hisds string 's'. */ +void hi_sdstolower(hisds s) { + int len = hi_sdslen(s), j; for (j = 0; j < len; j++) s[j] = tolower(s[j]); } -/* Apply toupper() to every character of the sds string 's'. */ -void sdstoupper(sds s) { - int len = sdslen(s), j; +/* Apply toupper() to every character of the hisds string 's'. */ +void hi_sdstoupper(hisds s) { + int len = hi_sdslen(s), j; for (j = 0; j < len; j++) s[j] = toupper(s[j]); } -/* Compare two sds strings s1 and s2 with memcmp(). +/* Compare two hisds strings s1 and s2 with memcmp(). * * Return value: * @@ -773,12 +778,12 @@ void sdstoupper(sds s) { * If two strings share exactly the same prefix, but one of the two has * additional characters, the longer string is considered to be greater than * the smaller one. */ -int sdscmp(const sds s1, const sds s2) { +int hi_sdscmp(const hisds s1, const hisds s2) { size_t l1, l2, minlen; int cmp; - l1 = sdslen(s1); - l2 = sdslen(s2); + l1 = hi_sdslen(s1); + l2 = hi_sdslen(s2); minlen = (l1 < l2) ? l1 : l2; cmp = memcmp(s1,s2,minlen); if (cmp == 0) return l1-l2; @@ -786,7 +791,7 @@ int sdscmp(const sds s1, const sds s2) { } /* Split 's' with separator in 'sep'. An array - * of sds strings is returned. *count will be set + * of hisds strings is returned. *count will be set * by reference to the number of tokens returned. * * On out of memory, zero length string, zero length @@ -794,20 +799,20 @@ int sdscmp(const sds s1, const sds s2) { * * Note that 'sep' is able to split a string using * a multi-character separator. For example - * sdssplit("foo_-_bar","_-_"); will return two + * hi_sdssplit("foo_-_bar","_-_"); will return two * elements "foo" and "bar". * * This version of the function is binary-safe but - * requires length arguments. sdssplit() is just the + * requires length arguments. hi_sdssplit() is just the * same function but for zero-terminated strings. */ -sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) { +hisds *hi_sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) { int elements = 0, slots = 5, start = 0, j; - sds *tokens; + hisds *tokens; if (seplen < 1 || len < 0) return NULL; - tokens = s_malloc(sizeof(sds)*slots); + tokens = hi_s_malloc(sizeof(hisds)*slots); if (tokens == NULL) return NULL; if (len == 0) { @@ -817,16 +822,16 @@ sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count for (j = 0; j < (len-(seplen-1)); j++) { /* make sure there is room for the next element and the final one */ if (slots < elements+2) { - sds *newtokens; + hisds *newtokens; slots *= 2; - newtokens = s_realloc(tokens,sizeof(sds)*slots); + newtokens = hi_s_realloc(tokens,sizeof(hisds)*slots); if (newtokens == NULL) goto cleanup; tokens = newtokens; } /* search the separator */ if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) { - tokens[elements] = sdsnewlen(s+start,j-start); + tokens[elements] = hi_sdsnewlen(s+start,j-start); if (tokens[elements] == NULL) goto cleanup; elements++; start = j+seplen; @@ -834,7 +839,7 @@ sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count } } /* Add the final element. We are sure there is room in the tokens array. */ - tokens[elements] = sdsnewlen(s+start,len-start); + tokens[elements] = hi_sdsnewlen(s+start,len-start); if (tokens[elements] == NULL) goto cleanup; elements++; *count = elements; @@ -843,62 +848,55 @@ sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count cleanup: { int i; - for (i = 0; i < elements; i++) sdsfree(tokens[i]); - s_free(tokens); + for (i = 0; i < elements; i++) hi_sdsfree(tokens[i]); + hi_s_free(tokens); *count = 0; return NULL; } } -/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */ -void sdsfreesplitres(sds *tokens, int count) { +/* Free the result returned by hi_sdssplitlen(), or do nothing if 'tokens' is NULL. */ +void hi_sdsfreesplitres(hisds *tokens, int count) { if (!tokens) return; while(count--) - sdsfree(tokens[count]); - s_free(tokens); + hi_sdsfree(tokens[count]); + hi_s_free(tokens); } -/* Append to the sds string "s" an escaped string representation where +/* Append to the hisds string "s" an escaped string representation where * all the non-printable characters (tested with isprint()) are turned into * escapes in the form "\n\r\a...." or "\x". * - * After the call, the modified sds string is no longer valid and all the + * After the call, the modified hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdscatrepr(sds s, const char *p, size_t len) { - s = sdscatlen(s,"\"",1); +hisds hi_sdscatrepr(hisds s, const char *p, size_t len) { + s = hi_sdscatlen(s,"\"",1); while(len--) { switch(*p) { case '\\': case '"': - s = sdscatprintf(s,"\\%c",*p); + s = hi_sdscatprintf(s,"\\%c",*p); break; - case '\n': s = sdscatlen(s,"\\n",2); break; - case '\r': s = sdscatlen(s,"\\r",2); break; - case '\t': s = sdscatlen(s,"\\t",2); break; - case '\a': s = sdscatlen(s,"\\a",2); break; - case '\b': s = sdscatlen(s,"\\b",2); break; + case '\n': s = hi_sdscatlen(s,"\\n",2); break; + case '\r': s = hi_sdscatlen(s,"\\r",2); break; + case '\t': s = hi_sdscatlen(s,"\\t",2); break; + case '\a': s = hi_sdscatlen(s,"\\a",2); break; + case '\b': s = hi_sdscatlen(s,"\\b",2); break; default: if (isprint(*p)) - s = sdscatprintf(s,"%c",*p); + s = hi_sdscatprintf(s,"%c",*p); else - s = sdscatprintf(s,"\\x%02x",(unsigned char)*p); + s = hi_sdscatprintf(s,"\\x%02x",(unsigned char)*p); break; } p++; } - return sdscatlen(s,"\"",1); + return hi_sdscatlen(s,"\"",1); } -/* Helper function for sdssplitargs() that returns non zero if 'c' - * is a valid hex digit. */ -int is_hex_digit(char c) { - return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || - (c >= 'A' && c <= 'F'); -} - -/* Helper function for sdssplitargs() that converts a hex digit into an +/* Helper function for hi_sdssplitargs() that converts a hex digit into an * integer from 0 to 15 */ -int hex_digit_to_int(char c) { +static int hi_hex_digit_to_int(char c) { switch(c) { case '0': return 0; case '1': return 1; @@ -926,20 +924,20 @@ int hex_digit_to_int(char c) { * foo bar "newline are supported\n" and "\xff\x00otherstuff" * * The number of arguments is stored into *argc, and an array - * of sds is returned. + * of hisds is returned. * - * The caller should free the resulting array of sds strings with - * sdsfreesplitres(). + * The caller should free the resulting array of hisds strings with + * hi_sdsfreesplitres(). * - * Note that sdscatrepr() is able to convert back a string into - * a quoted string in the same format sdssplitargs() is able to parse. + * Note that hi_sdscatrepr() is able to convert back a string into + * a quoted string in the same format hi_sdssplitargs() is able to parse. * * The function returns the allocated tokens on success, even when the * input string is empty, or NULL if the input contains unbalanced * quotes or closed quotes followed by non space characters * as in: "foo"bar or "foo' */ -sds *sdssplitargs(const char *line, int *argc) { +hisds *hi_sdssplitargs(const char *line, int *argc) { const char *p = line; char *current = NULL; char **vector = NULL; @@ -954,18 +952,18 @@ sds *sdssplitargs(const char *line, int *argc) { int insq=0; /* set to 1 if we are in 'single quotes' */ int done=0; - if (current == NULL) current = sdsempty(); + if (current == NULL) current = hi_sdsempty(); while(!done) { if (inq) { if (*p == '\\' && *(p+1) == 'x' && - is_hex_digit(*(p+2)) && - is_hex_digit(*(p+3))) + isxdigit(*(p+2)) && + isxdigit(*(p+3))) { unsigned char byte; - byte = (hex_digit_to_int(*(p+2))*16)+ - hex_digit_to_int(*(p+3)); - current = sdscatlen(current,(char*)&byte,1); + byte = (hi_hex_digit_to_int(*(p+2))*16)+ + hi_hex_digit_to_int(*(p+3)); + current = hi_sdscatlen(current,(char*)&byte,1); p += 3; } else if (*p == '\\' && *(p+1)) { char c; @@ -979,7 +977,7 @@ sds *sdssplitargs(const char *line, int *argc) { case 'a': c = '\a'; break; default: c = *p; break; } - current = sdscatlen(current,&c,1); + current = hi_sdscatlen(current,&c,1); } else if (*p == '"') { /* closing quote must be followed by a space or * nothing at all. */ @@ -989,12 +987,12 @@ sds *sdssplitargs(const char *line, int *argc) { /* unterminated quotes */ goto err; } else { - current = sdscatlen(current,p,1); + current = hi_sdscatlen(current,p,1); } } else if (insq) { if (*p == '\\' && *(p+1) == '\'') { p++; - current = sdscatlen(current,"'",1); + current = hi_sdscatlen(current,"'",1); } else if (*p == '\'') { /* closing quote must be followed by a space or * nothing at all. */ @@ -1004,7 +1002,7 @@ sds *sdssplitargs(const char *line, int *argc) { /* unterminated quotes */ goto err; } else { - current = sdscatlen(current,p,1); + current = hi_sdscatlen(current,p,1); } } else { switch(*p) { @@ -1022,7 +1020,7 @@ sds *sdssplitargs(const char *line, int *argc) { insq=1; break; default: - current = sdscatlen(current,p,1); + current = hi_sdscatlen(current,p,1); break; } } @@ -1030,9 +1028,9 @@ sds *sdssplitargs(const char *line, int *argc) { } /* add the token to the vector */ { - char **new_vector = s_realloc(vector,((*argc)+1)*sizeof(char*)); + char **new_vector = hi_s_realloc(vector,((*argc)+1)*sizeof(char*)); if (new_vector == NULL) { - s_free(vector); + hi_s_free(vector); return NULL; } @@ -1043,16 +1041,16 @@ sds *sdssplitargs(const char *line, int *argc) { } } else { /* Even on empty input string return something not NULL. */ - if (vector == NULL) vector = s_malloc(sizeof(void*)); + if (vector == NULL) vector = hi_s_malloc(sizeof(void*)); return vector; } } err: while((*argc)--) - sdsfree(vector[*argc]); - s_free(vector); - if (current) sdsfree(current); + hi_sdsfree(vector[*argc]); + hi_s_free(vector); + if (current) hi_sdsfree(current); *argc = 0; return NULL; } @@ -1061,13 +1059,13 @@ err: * characters specified in the 'from' string to the corresponding character * in the 'to' array. * - * For instance: sdsmapchars(mystring, "ho", "01", 2) + * For instance: hi_sdsmapchars(mystring, "ho", "01", 2) * will have the effect of turning the string "hello" into "0ell1". * - * The function returns the sds string pointer, that is always the same + * The function returns the hisds string pointer, that is always the same * as the input pointer since no resize is needed. */ -sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) { - size_t j, i, l = sdslen(s); +hisds hi_sdsmapchars(hisds s, const char *from, const char *to, size_t setlen) { + size_t j, i, l = hi_sdslen(s); for (j = 0; j < l; j++) { for (i = 0; i < setlen; i++) { @@ -1081,26 +1079,26 @@ sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) { } /* Join an array of C strings using the specified separator (also a C string). - * Returns the result as an sds string. */ -sds sdsjoin(char **argv, int argc, char *sep) { - sds join = sdsempty(); + * Returns the result as an hisds string. */ +hisds hi_sdsjoin(char **argv, int argc, char *sep) { + hisds join = hi_sdsempty(); int j; for (j = 0; j < argc; j++) { - join = sdscat(join, argv[j]); - if (j != argc-1) join = sdscat(join,sep); + join = hi_sdscat(join, argv[j]); + if (j != argc-1) join = hi_sdscat(join,sep); } return join; } -/* Like sdsjoin, but joins an array of SDS strings. */ -sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) { - sds join = sdsempty(); +/* Like hi_sdsjoin, but joins an array of SDS strings. */ +hisds hi_sdsjoinsds(hisds *argv, int argc, const char *sep, size_t seplen) { + hisds join = hi_sdsempty(); int j; for (j = 0; j < argc; j++) { - join = sdscatsds(join, argv[j]); - if (j != argc-1) join = sdscatlen(join,sep,seplen); + join = hi_sdscatsds(join, argv[j]); + if (j != argc-1) join = hi_sdscatlen(join,sep,seplen); } return join; } @@ -1110,138 +1108,138 @@ sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) { * the overhead of function calls. Here we define these wrappers only for * the programs SDS is linked to, if they want to touch the SDS internals * even if they use a different allocator. */ -void *sds_malloc(size_t size) { return s_malloc(size); } -void *sds_realloc(void *ptr, size_t size) { return s_realloc(ptr,size); } -void sds_free(void *ptr) { s_free(ptr); } +void *hi_sds_malloc(size_t size) { return hi_s_malloc(size); } +void *hi_sds_realloc(void *ptr, size_t size) { return hi_s_realloc(ptr,size); } +void hi_sds_free(void *ptr) { hi_s_free(ptr); } -#if defined(SDS_TEST_MAIN) +#if defined(HI_SDS_TEST_MAIN) #include #include "testhelp.h" #include "limits.h" #define UNUSED(x) (void)(x) -int sdsTest(void) { +int hi_sdsTest(void) { { - sds x = sdsnew("foo"), y; + hisds x = hi_sdsnew("foo"), y; test_cond("Create a string and obtain the length", - sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0) + hi_sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0) - sdsfree(x); - x = sdsnewlen("foo",2); + hi_sdsfree(x); + x = hi_sdsnewlen("foo",2); test_cond("Create a string with specified length", - sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0) + hi_sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0) - x = sdscat(x,"bar"); + x = hi_sdscat(x,"bar"); test_cond("Strings concatenation", - sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0); + hi_sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0); - x = sdscpy(x,"a"); - test_cond("sdscpy() against an originally longer string", - sdslen(x) == 1 && memcmp(x,"a\0",2) == 0) + x = hi_sdscpy(x,"a"); + test_cond("hi_sdscpy() against an originally longer string", + hi_sdslen(x) == 1 && memcmp(x,"a\0",2) == 0) - x = sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk"); - test_cond("sdscpy() against an originally shorter string", - sdslen(x) == 33 && + x = hi_sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk"); + test_cond("hi_sdscpy() against an originally shorter string", + hi_sdslen(x) == 33 && memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0) - sdsfree(x); - x = sdscatprintf(sdsempty(),"%d",123); - test_cond("sdscatprintf() seems working in the base case", - sdslen(x) == 3 && memcmp(x,"123\0",4) == 0) + hi_sdsfree(x); + x = hi_sdscatprintf(hi_sdsempty(),"%d",123); + test_cond("hi_sdscatprintf() seems working in the base case", + hi_sdslen(x) == 3 && memcmp(x,"123\0",4) == 0) - sdsfree(x); - x = sdsnew("--"); - x = sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX); - test_cond("sdscatfmt() seems working in the base case", - sdslen(x) == 60 && + hi_sdsfree(x); + x = hi_sdsnew("--"); + x = hi_sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX); + test_cond("hi_sdscatfmt() seems working in the base case", + hi_sdslen(x) == 60 && memcmp(x,"--Hello Hi! World -9223372036854775808," "9223372036854775807--",60) == 0) printf("[%s]\n",x); - sdsfree(x); - x = sdsnew("--"); - x = sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX); - test_cond("sdscatfmt() seems working with unsigned numbers", - sdslen(x) == 35 && + hi_sdsfree(x); + x = hi_sdsnew("--"); + x = hi_sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX); + test_cond("hi_sdscatfmt() seems working with unsigned numbers", + hi_sdslen(x) == 35 && memcmp(x,"--4294967295,18446744073709551615--",35) == 0) - sdsfree(x); - x = sdsnew(" x "); - sdstrim(x," x"); - test_cond("sdstrim() works when all chars match", - sdslen(x) == 0) + hi_sdsfree(x); + x = hi_sdsnew(" x "); + hi_sdstrim(x," x"); + test_cond("hi_sdstrim() works when all chars match", + hi_sdslen(x) == 0) - sdsfree(x); - x = sdsnew(" x "); - sdstrim(x," "); - test_cond("sdstrim() works when a single char remains", - sdslen(x) == 1 && x[0] == 'x') + hi_sdsfree(x); + x = hi_sdsnew(" x "); + hi_sdstrim(x," "); + test_cond("hi_sdstrim() works when a single char remains", + hi_sdslen(x) == 1 && x[0] == 'x') - sdsfree(x); - x = sdsnew("xxciaoyyy"); - sdstrim(x,"xy"); - test_cond("sdstrim() correctly trims characters", - sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0) + hi_sdsfree(x); + x = hi_sdsnew("xxciaoyyy"); + hi_sdstrim(x,"xy"); + test_cond("hi_sdstrim() correctly trims characters", + hi_sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0) - y = sdsdup(x); - sdsrange(y,1,1); - test_cond("sdsrange(...,1,1)", - sdslen(y) == 1 && memcmp(y,"i\0",2) == 0) + y = hi_sdsdup(x); + hi_sdsrange(y,1,1); + test_cond("hi_sdsrange(...,1,1)", + hi_sdslen(y) == 1 && memcmp(y,"i\0",2) == 0) - sdsfree(y); - y = sdsdup(x); - sdsrange(y,1,-1); - test_cond("sdsrange(...,1,-1)", - sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,1,-1); + test_cond("hi_sdsrange(...,1,-1)", + hi_sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) - sdsfree(y); - y = sdsdup(x); - sdsrange(y,-2,-1); - test_cond("sdsrange(...,-2,-1)", - sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0) + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,-2,-1); + test_cond("hi_sdsrange(...,-2,-1)", + hi_sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0) - sdsfree(y); - y = sdsdup(x); - sdsrange(y,2,1); - test_cond("sdsrange(...,2,1)", - sdslen(y) == 0 && memcmp(y,"\0",1) == 0) + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,2,1); + test_cond("hi_sdsrange(...,2,1)", + hi_sdslen(y) == 0 && memcmp(y,"\0",1) == 0) - sdsfree(y); - y = sdsdup(x); - sdsrange(y,1,100); - test_cond("sdsrange(...,1,100)", - sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,1,100); + test_cond("hi_sdsrange(...,1,100)", + hi_sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) - sdsfree(y); - y = sdsdup(x); - sdsrange(y,100,100); - test_cond("sdsrange(...,100,100)", - sdslen(y) == 0 && memcmp(y,"\0",1) == 0) + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,100,100); + test_cond("hi_sdsrange(...,100,100)", + hi_sdslen(y) == 0 && memcmp(y,"\0",1) == 0) - sdsfree(y); - sdsfree(x); - x = sdsnew("foo"); - y = sdsnew("foa"); - test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0) + hi_sdsfree(y); + hi_sdsfree(x); + x = hi_sdsnew("foo"); + y = hi_sdsnew("foa"); + test_cond("hi_sdscmp(foo,foa)", hi_sdscmp(x,y) > 0) - sdsfree(y); - sdsfree(x); - x = sdsnew("bar"); - y = sdsnew("bar"); - test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0) + hi_sdsfree(y); + hi_sdsfree(x); + x = hi_sdsnew("bar"); + y = hi_sdsnew("bar"); + test_cond("hi_sdscmp(bar,bar)", hi_sdscmp(x,y) == 0) - sdsfree(y); - sdsfree(x); - x = sdsnew("aar"); - y = sdsnew("bar"); - test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0) + hi_sdsfree(y); + hi_sdsfree(x); + x = hi_sdsnew("aar"); + y = hi_sdsnew("bar"); + test_cond("hi_sdscmp(bar,bar)", hi_sdscmp(x,y) < 0) - sdsfree(y); - sdsfree(x); - x = sdsnewlen("\a\n\0foo\r",7); - y = sdscatrepr(sdsempty(),x,sdslen(x)); - test_cond("sdscatrepr(...data...)", + hi_sdsfree(y); + hi_sdsfree(x); + x = hi_sdsnewlen("\a\n\0foo\r",7); + y = hi_sdscatrepr(hi_sdsempty(),x,hi_sdslen(x)); + test_cond("hi_sdscatrepr(...data...)", memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0) { @@ -1249,43 +1247,43 @@ int sdsTest(void) { char *p; int step = 10, j, i; - sdsfree(x); - sdsfree(y); - x = sdsnew("0"); - test_cond("sdsnew() free/len buffers", sdslen(x) == 1 && sdsavail(x) == 0); + hi_sdsfree(x); + hi_sdsfree(y); + x = hi_sdsnew("0"); + test_cond("hi_sdsnew() free/len buffers", hi_sdslen(x) == 1 && hi_sdsavail(x) == 0); /* Run the test a few times in order to hit the first two * SDS header types. */ for (i = 0; i < 10; i++) { - int oldlen = sdslen(x); - x = sdsMakeRoomFor(x,step); - int type = x[-1]&SDS_TYPE_MASK; + int oldlen = hi_sdslen(x); + x = hi_sdsMakeRoomFor(x,step); + int type = x[-1]&HI_SDS_TYPE_MASK; - test_cond("sdsMakeRoomFor() len", sdslen(x) == oldlen); - if (type != SDS_TYPE_5) { - test_cond("sdsMakeRoomFor() free", sdsavail(x) >= step); - oldfree = sdsavail(x); + test_cond("sdsMakeRoomFor() len", hi_sdslen(x) == oldlen); + if (type != HI_SDS_TYPE_5) { + test_cond("hi_sdsMakeRoomFor() free", hi_sdsavail(x) >= step); + oldfree = hi_sdsavail(x); } p = x+oldlen; for (j = 0; j < step; j++) { p[j] = 'A'+j; } - sdsIncrLen(x,step); + hi_sdsIncrLen(x,step); } - test_cond("sdsMakeRoomFor() content", + test_cond("hi_sdsMakeRoomFor() content", memcmp("0ABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJ",x,101) == 0); - test_cond("sdsMakeRoomFor() final length",sdslen(x)==101); + test_cond("sdsMakeRoomFor() final length",hi_sdslen(x)==101); - sdsfree(x); + hi_sdsfree(x); } } - test_report() + test_report(); return 0; } #endif -#ifdef SDS_TEST_MAIN +#ifdef HI_SDS_TEST_MAIN int main(void) { - return sdsTest(); + return hi_sdsTest(); } #endif diff --git a/deps/hiredis/sds.h b/deps/hiredis/sds.h index 0ce33b1fc..6aadf8ea2 100644 --- a/deps/hiredis/sds.h +++ b/deps/hiredis/sds.h @@ -30,29 +30,31 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#ifndef __SDS_H -#define __SDS_H +#ifndef HIREDIS_SDS_H +#define HIREDIS_SDS_H -#define SDS_MAX_PREALLOC (1024*1024) +#define HI_SDS_MAX_PREALLOC (1024*1024) #ifdef _MSC_VER #define __attribute__(x) +typedef long long ssize_t; +#define SSIZE_MAX (LLONG_MAX >> 1) #endif #include #include #include -typedef char *sds; +typedef char *hisds; /* Note: sdshdr5 is never used, we just access the flags byte directly. * However is here to document the layout of type 5 SDS strings. */ -struct __attribute__ ((__packed__)) sdshdr5 { +struct __attribute__ ((__packed__)) hisdshdr5 { unsigned char flags; /* 3 lsb of type, and 5 msb of string length */ #ifndef __cplusplus char buf[]; #endif }; -struct __attribute__ ((__packed__)) sdshdr8 { +struct __attribute__ ((__packed__)) hisdshdr8 { uint8_t len; /* used */ uint8_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ @@ -60,7 +62,7 @@ struct __attribute__ ((__packed__)) sdshdr8 { char buf[]; #endif }; -struct __attribute__ ((__packed__)) sdshdr16 { +struct __attribute__ ((__packed__)) hisdshdr16 { uint16_t len; /* used */ uint16_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ @@ -68,7 +70,7 @@ struct __attribute__ ((__packed__)) sdshdr16 { char buf[]; #endif }; -struct __attribute__ ((__packed__)) sdshdr32 { +struct __attribute__ ((__packed__)) hisdshdr32 { uint32_t len; /* used */ uint32_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ @@ -76,7 +78,7 @@ struct __attribute__ ((__packed__)) sdshdr32 { char buf[]; #endif }; -struct __attribute__ ((__packed__)) sdshdr64 { +struct __attribute__ ((__packed__)) hisdshdr64 { uint64_t len; /* used */ uint64_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ @@ -85,203 +87,203 @@ struct __attribute__ ((__packed__)) sdshdr64 { #endif }; -#define SDS_TYPE_5 0 -#define SDS_TYPE_8 1 -#define SDS_TYPE_16 2 -#define SDS_TYPE_32 3 -#define SDS_TYPE_64 4 -#define SDS_TYPE_MASK 7 -#define SDS_TYPE_BITS 3 -#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))); -#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)))) -#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS) +#define HI_SDS_TYPE_5 0 +#define HI_SDS_TYPE_8 1 +#define HI_SDS_TYPE_16 2 +#define HI_SDS_TYPE_32 3 +#define HI_SDS_TYPE_64 4 +#define HI_SDS_TYPE_MASK 7 +#define HI_SDS_TYPE_BITS 3 +#define HI_SDS_HDR_VAR(T,s) struct hisdshdr##T *sh = (struct hisdshdr##T *)((s)-(sizeof(struct hisdshdr##T))); +#define HI_SDS_HDR(T,s) ((struct hisdshdr##T *)((s)-(sizeof(struct hisdshdr##T)))) +#define HI_SDS_TYPE_5_LEN(f) ((f)>>HI_SDS_TYPE_BITS) -static inline size_t sdslen(const sds s) { +static inline size_t hi_sdslen(const hisds s) { unsigned char flags = s[-1]; - switch(__builtin_expect((flags&SDS_TYPE_MASK), SDS_TYPE_5)) { - case SDS_TYPE_5: - return SDS_TYPE_5_LEN(flags); - case SDS_TYPE_8: - return SDS_HDR(8,s)->len; - case SDS_TYPE_16: - return SDS_HDR(16,s)->len; - case SDS_TYPE_32: - return SDS_HDR(32,s)->len; - case SDS_TYPE_64: - return SDS_HDR(64,s)->len; + switch(__builtin_expect((flags&HI_SDS_TYPE_MASK), HI_SDS_TYPE_5)) { + case HI_SDS_TYPE_5: + return HI_SDS_TYPE_5_LEN(flags); + case HI_SDS_TYPE_8: + return HI_SDS_HDR(8,s)->len; + case HI_SDS_TYPE_16: + return HI_SDS_HDR(16,s)->len; + case HI_SDS_TYPE_32: + return HI_SDS_HDR(32,s)->len; + case HI_SDS_TYPE_64: + return HI_SDS_HDR(64,s)->len; } return 0; } -static inline size_t sdsavail(const sds s) { +static inline size_t hi_sdsavail(const hisds s) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: { + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: { return 0; } - case SDS_TYPE_8: { - SDS_HDR_VAR(8,s); + case HI_SDS_TYPE_8: { + HI_SDS_HDR_VAR(8,s); return sh->alloc - sh->len; } - case SDS_TYPE_16: { - SDS_HDR_VAR(16,s); + case HI_SDS_TYPE_16: { + HI_SDS_HDR_VAR(16,s); return sh->alloc - sh->len; } - case SDS_TYPE_32: { - SDS_HDR_VAR(32,s); + case HI_SDS_TYPE_32: { + HI_SDS_HDR_VAR(32,s); return sh->alloc - sh->len; } - case SDS_TYPE_64: { - SDS_HDR_VAR(64,s); + case HI_SDS_TYPE_64: { + HI_SDS_HDR_VAR(64,s); return sh->alloc - sh->len; } } return 0; } -static inline void sdssetlen(sds s, size_t newlen) { +static inline void hi_sdssetlen(hisds s, size_t newlen) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - *fp = (unsigned char)(SDS_TYPE_5 | (newlen << SDS_TYPE_BITS)); + *fp = (unsigned char)(HI_SDS_TYPE_5 | (newlen << HI_SDS_TYPE_BITS)); } break; - case SDS_TYPE_8: - SDS_HDR(8,s)->len = (uint8_t)newlen; + case HI_SDS_TYPE_8: + HI_SDS_HDR(8,s)->len = (uint8_t)newlen; break; - case SDS_TYPE_16: - SDS_HDR(16,s)->len = (uint16_t)newlen; + case HI_SDS_TYPE_16: + HI_SDS_HDR(16,s)->len = (uint16_t)newlen; break; - case SDS_TYPE_32: - SDS_HDR(32,s)->len = (uint32_t)newlen; + case HI_SDS_TYPE_32: + HI_SDS_HDR(32,s)->len = (uint32_t)newlen; break; - case SDS_TYPE_64: - SDS_HDR(64,s)->len = (uint64_t)newlen; + case HI_SDS_TYPE_64: + HI_SDS_HDR(64,s)->len = (uint64_t)newlen; break; } } -static inline void sdsinclen(sds s, size_t inc) { +static inline void hi_sdsinclen(hisds s, size_t inc) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - unsigned char newlen = SDS_TYPE_5_LEN(flags)+(unsigned char)inc; - *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); + unsigned char newlen = HI_SDS_TYPE_5_LEN(flags)+(unsigned char)inc; + *fp = HI_SDS_TYPE_5 | (newlen << HI_SDS_TYPE_BITS); } break; - case SDS_TYPE_8: - SDS_HDR(8,s)->len += (uint8_t)inc; + case HI_SDS_TYPE_8: + HI_SDS_HDR(8,s)->len += (uint8_t)inc; break; - case SDS_TYPE_16: - SDS_HDR(16,s)->len += (uint16_t)inc; + case HI_SDS_TYPE_16: + HI_SDS_HDR(16,s)->len += (uint16_t)inc; break; - case SDS_TYPE_32: - SDS_HDR(32,s)->len += (uint32_t)inc; + case HI_SDS_TYPE_32: + HI_SDS_HDR(32,s)->len += (uint32_t)inc; break; - case SDS_TYPE_64: - SDS_HDR(64,s)->len += (uint64_t)inc; + case HI_SDS_TYPE_64: + HI_SDS_HDR(64,s)->len += (uint64_t)inc; break; } } -/* sdsalloc() = sdsavail() + sdslen() */ -static inline size_t sdsalloc(const sds s) { +/* hi_sdsalloc() = hi_sdsavail() + hi_sdslen() */ +static inline size_t hi_sdsalloc(const hisds s) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: - return SDS_TYPE_5_LEN(flags); - case SDS_TYPE_8: - return SDS_HDR(8,s)->alloc; - case SDS_TYPE_16: - return SDS_HDR(16,s)->alloc; - case SDS_TYPE_32: - return SDS_HDR(32,s)->alloc; - case SDS_TYPE_64: - return SDS_HDR(64,s)->alloc; + switch(flags & HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: + return HI_SDS_TYPE_5_LEN(flags); + case HI_SDS_TYPE_8: + return HI_SDS_HDR(8,s)->alloc; + case HI_SDS_TYPE_16: + return HI_SDS_HDR(16,s)->alloc; + case HI_SDS_TYPE_32: + return HI_SDS_HDR(32,s)->alloc; + case HI_SDS_TYPE_64: + return HI_SDS_HDR(64,s)->alloc; } return 0; } -static inline void sdssetalloc(sds s, size_t newlen) { +static inline void hi_sdssetalloc(hisds s, size_t newlen) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: /* Nothing to do, this type has no total allocation info. */ break; - case SDS_TYPE_8: - SDS_HDR(8,s)->alloc = (uint8_t)newlen; + case HI_SDS_TYPE_8: + HI_SDS_HDR(8,s)->alloc = (uint8_t)newlen; break; - case SDS_TYPE_16: - SDS_HDR(16,s)->alloc = (uint16_t)newlen; + case HI_SDS_TYPE_16: + HI_SDS_HDR(16,s)->alloc = (uint16_t)newlen; break; - case SDS_TYPE_32: - SDS_HDR(32,s)->alloc = (uint32_t)newlen; + case HI_SDS_TYPE_32: + HI_SDS_HDR(32,s)->alloc = (uint32_t)newlen; break; - case SDS_TYPE_64: - SDS_HDR(64,s)->alloc = (uint64_t)newlen; + case HI_SDS_TYPE_64: + HI_SDS_HDR(64,s)->alloc = (uint64_t)newlen; break; } } -sds sdsnewlen(const void *init, size_t initlen); -sds sdsnew(const char *init); -sds sdsempty(void); -sds sdsdup(const sds s); -void sdsfree(sds s); -sds sdsgrowzero(sds s, size_t len); -sds sdscatlen(sds s, const void *t, size_t len); -sds sdscat(sds s, const char *t); -sds sdscatsds(sds s, const sds t); -sds sdscpylen(sds s, const char *t, size_t len); -sds sdscpy(sds s, const char *t); +hisds hi_sdsnewlen(const void *init, size_t initlen); +hisds hi_sdsnew(const char *init); +hisds hi_sdsempty(void); +hisds hi_sdsdup(const hisds s); +void hi_sdsfree(hisds s); +hisds hi_sdsgrowzero(hisds s, size_t len); +hisds hi_sdscatlen(hisds s, const void *t, size_t len); +hisds hi_sdscat(hisds s, const char *t); +hisds hi_sdscatsds(hisds s, const hisds t); +hisds hi_sdscpylen(hisds s, const char *t, size_t len); +hisds hi_sdscpy(hisds s, const char *t); -sds sdscatvprintf(sds s, const char *fmt, va_list ap); +hisds hi_sdscatvprintf(hisds s, const char *fmt, va_list ap); #ifdef __GNUC__ -sds sdscatprintf(sds s, const char *fmt, ...) +hisds hi_sdscatprintf(hisds s, const char *fmt, ...) __attribute__((format(printf, 2, 3))); #else -sds sdscatprintf(sds s, const char *fmt, ...); +hisds hi_sdscatprintf(hisds s, const char *fmt, ...); #endif -sds sdscatfmt(sds s, char const *fmt, ...); -sds sdstrim(sds s, const char *cset); -void sdsrange(sds s, int start, int end); -void sdsupdatelen(sds s); -void sdsclear(sds s); -int sdscmp(const sds s1, const sds s2); -sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count); -void sdsfreesplitres(sds *tokens, int count); -void sdstolower(sds s); -void sdstoupper(sds s); -sds sdsfromlonglong(long long value); -sds sdscatrepr(sds s, const char *p, size_t len); -sds *sdssplitargs(const char *line, int *argc); -sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen); -sds sdsjoin(char **argv, int argc, char *sep); -sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen); +hisds hi_sdscatfmt(hisds s, char const *fmt, ...); +hisds hi_sdstrim(hisds s, const char *cset); +int hi_sdsrange(hisds s, ssize_t start, ssize_t end); +void hi_sdsupdatelen(hisds s); +void hi_sdsclear(hisds s); +int hi_sdscmp(const hisds s1, const hisds s2); +hisds *hi_sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count); +void hi_sdsfreesplitres(hisds *tokens, int count); +void hi_sdstolower(hisds s); +void hi_sdstoupper(hisds s); +hisds hi_sdsfromlonglong(long long value); +hisds hi_sdscatrepr(hisds s, const char *p, size_t len); +hisds *hi_sdssplitargs(const char *line, int *argc); +hisds hi_sdsmapchars(hisds s, const char *from, const char *to, size_t setlen); +hisds hi_sdsjoin(char **argv, int argc, char *sep); +hisds hi_sdsjoinsds(hisds *argv, int argc, const char *sep, size_t seplen); /* Low level functions exposed to the user API */ -sds sdsMakeRoomFor(sds s, size_t addlen); -void sdsIncrLen(sds s, int incr); -sds sdsRemoveFreeSpace(sds s); -size_t sdsAllocSize(sds s); -void *sdsAllocPtr(sds s); +hisds hi_sdsMakeRoomFor(hisds s, size_t addlen); +void hi_sdsIncrLen(hisds s, int incr); +hisds hi_sdsRemoveFreeSpace(hisds s); +size_t hi_sdsAllocSize(hisds s); +void *hi_sdsAllocPtr(hisds s); /* Export the allocator used by SDS to the program using SDS. * Sometimes the program SDS is linked to, may use a different set of * allocators, but may want to allocate or free things that SDS will * respectively free or allocate. */ -void *sds_malloc(size_t size); -void *sds_realloc(void *ptr, size_t size); -void sds_free(void *ptr); +void *hi_sds_malloc(size_t size); +void *hi_sds_realloc(void *ptr, size_t size); +void hi_sds_free(void *ptr); #ifdef REDIS_TEST -int sdsTest(int argc, char *argv[]); +int hi_sdsTest(int argc, char *argv[]); #endif -#endif +#endif /* HIREDIS_SDS_H */ diff --git a/deps/hiredis/sdsalloc.h b/deps/hiredis/sdsalloc.h index f43023c48..c9dcc3df8 100644 --- a/deps/hiredis/sdsalloc.h +++ b/deps/hiredis/sdsalloc.h @@ -37,6 +37,8 @@ * the include of your alternate allocator if needed (not needed in order * to use the default libc allocator). */ -#define s_malloc malloc -#define s_realloc realloc -#define s_free free +#include "alloc.h" + +#define hi_s_malloc hi_malloc +#define hi_s_realloc hi_realloc +#define hi_s_free hi_free diff --git a/deps/hiredis/sdscompat.h b/deps/hiredis/sdscompat.h new file mode 100644 index 000000000..e5a2574f3 --- /dev/null +++ b/deps/hiredis/sdscompat.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2020, Michael Grunder + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * SDS compatibility header. + * + * This simple file maps sds types and calls to their unique hiredis symbol names. + * It's useful when we build Hiredis as a dependency of Redis and want to call + * Hiredis' sds symbols rather than the ones built into Redis, as the libraries + * have slightly diverged and could cause hard to track down ABI incompatibility + * bugs. + * + */ + +#ifndef HIREDIS_SDS_COMPAT +#define HIREDIS_SDS_COMPAT + +#define sds hisds + +#define sdslen hi_sdslen +#define sdsavail hi_sdsavail +#define sdssetlen hi_sdssetlen +#define sdsinclen hi_sdsinclen +#define sdsalloc hi_sdsalloc +#define sdssetalloc hi_sdssetalloc + +#define sdsAllocPtr hi_sdsAllocPtr +#define sdsAllocSize hi_sdsAllocSize +#define sdscat hi_sdscat +#define sdscatfmt hi_sdscatfmt +#define sdscatlen hi_sdscatlen +#define sdscatprintf hi_sdscatprintf +#define sdscatrepr hi_sdscatrepr +#define sdscatsds hi_sdscatsds +#define sdscatvprintf hi_sdscatvprintf +#define sdsclear hi_sdsclear +#define sdscmp hi_sdscmp +#define sdscpy hi_sdscpy +#define sdscpylen hi_sdscpylen +#define sdsdup hi_sdsdup +#define sdsempty hi_sdsempty +#define sds_free hi_sds_free +#define sdsfree hi_sdsfree +#define sdsfreesplitres hi_sdsfreesplitres +#define sdsfromlonglong hi_sdsfromlonglong +#define sdsgrowzero hi_sdsgrowzero +#define sdsIncrLen hi_sdsIncrLen +#define sdsjoin hi_sdsjoin +#define sdsjoinsds hi_sdsjoinsds +#define sdsll2str hi_sdsll2str +#define sdsMakeRoomFor hi_sdsMakeRoomFor +#define sds_malloc hi_sds_malloc +#define sdsmapchars hi_sdsmapchars +#define sdsnew hi_sdsnew +#define sdsnewlen hi_sdsnewlen +#define sdsrange hi_sdsrange +#define sds_realloc hi_sds_realloc +#define sdsRemoveFreeSpace hi_sdsRemoveFreeSpace +#define sdssplitargs hi_sdssplitargs +#define sdssplitlen hi_sdssplitlen +#define sdstolower hi_sdstolower +#define sdstoupper hi_sdstoupper +#define sdstrim hi_sdstrim +#define sdsull2str hi_sdsull2str +#define sdsupdatelen hi_sdsupdatelen + +#endif /* HIREDIS_SDS_COMPAT */ diff --git a/deps/hiredis/sockcompat.c b/deps/hiredis/sockcompat.c index 4cc2f414f..f99d14b05 100644 --- a/deps/hiredis/sockcompat.c +++ b/deps/hiredis/sockcompat.c @@ -212,7 +212,7 @@ int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, sockle int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen) { int ret = 0; if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { - struct timeval *tv = optval; + const struct timeval *tv = optval; DWORD timeout = tv->tv_sec * 1000 + tv->tv_usec / 1000; ret = setsockopt(sockfd, level, optname, (const char*)&timeout, sizeof(DWORD)); } else { diff --git a/deps/hiredis/sockcompat.h b/deps/hiredis/sockcompat.h index 56006c163..85810e848 100644 --- a/deps/hiredis/sockcompat.h +++ b/deps/hiredis/sockcompat.h @@ -49,9 +49,10 @@ #include #include #include +#include #ifdef _MSC_VER -typedef signed long ssize_t; +typedef long long ssize_t; #endif /* Emulate the parts of the BSD socket API that we need (override the winsock signatures). */ diff --git a/deps/hiredis/ssl.c b/deps/hiredis/ssl.c index 78ab9e43e..fe9a2fdce 100644 --- a/deps/hiredis/ssl.c +++ b/deps/hiredis/ssl.c @@ -34,25 +34,33 @@ #include "async.h" #include -#include #include #include +#ifdef _WIN32 +#include +#else +#include +#endif #include #include +#include "win32.h" #include "async_private.h" +#include "hiredis_ssl.h" void __redisSetError(redisContext *c, int type, const char *str); -/* The SSL context is attached to SSL/TLS connections as a privdata. */ -typedef struct redisSSLContext { - /** - * OpenSSL SSL_CTX; It is optional and will not be set when using - * user-supplied SSL. - */ +struct redisSSLContext { + /* Associated OpenSSL SSL_CTX as created by redisCreateSSLContext() */ SSL_CTX *ssl_ctx; + /* Requested SNI, or NULL */ + char *server_name; +}; + +/* The SSL connection context is attached to SSL/TLS connections as a privdata. */ +typedef struct redisSSL { /** * OpenSSL SSL object. */ @@ -72,43 +80,11 @@ typedef struct redisSSLContext { * should resume whenever a read takes place, if possible */ int pendingWrite; -} redisSSLContext; +} redisSSL; /* Forward declaration */ redisContextFuncs redisContextSSLFuncs; -#ifdef HIREDIS_SSL_TRACE -/** - * Callback used for debugging - */ -static void sslLogCallback(const SSL *ssl, int where, int ret) { - const char *retstr = ""; - int should_log = 1; - /* Ignore low-level SSL stuff */ - - if (where & SSL_CB_ALERT) { - should_log = 1; - } - if (where == SSL_CB_HANDSHAKE_START || where == SSL_CB_HANDSHAKE_DONE) { - should_log = 1; - } - if ((where & SSL_CB_EXIT) && ret == 0) { - should_log = 1; - } - - if (!should_log) { - return; - } - - retstr = SSL_alert_type_string(ret); - printf("ST(0x%x). %s. R(0x%x)%s\n", where, SSL_state_string_long(ssl), ret, retstr); - - if (where == SSL_CB_HANDSHAKE_DONE) { - printf("Using SSL version %s. Cipher=%s\n", SSL_get_version(ssl), SSL_get_cipher_name(ssl)); - } -} -#endif - /** * OpenSSL global initialization and locking handling callbacks. * Note that this is only required for OpenSSL < 1.1.0. @@ -119,6 +95,18 @@ static void sslLogCallback(const SSL *ssl, int where, int ret) { #endif #ifdef HIREDIS_USE_CRYPTO_LOCKS +#ifdef _WIN32 +typedef CRITICAL_SECTION sslLockType; +static void sslLockInit(sslLockType* l) { + InitializeCriticalSection(l); +} +static void sslLockAcquire(sslLockType* l) { + EnterCriticalSection(l); +} +static void sslLockRelease(sslLockType* l) { + LeaveCriticalSection(l); +} +#else typedef pthread_mutex_t sslLockType; static void sslLockInit(sslLockType *l) { pthread_mutex_init(l, NULL); @@ -129,7 +117,9 @@ static void sslLockAcquire(sslLockType *l) { static void sslLockRelease(sslLockType *l) { pthread_mutex_unlock(l); } -static pthread_mutex_t *ossl_locks; +#endif + +static sslLockType* ossl_locks; static void opensslDoLock(int mode, int lkid, const char *f, int line) { sslLockType *l = ossl_locks + lkid; @@ -144,36 +134,151 @@ static void opensslDoLock(int mode, int lkid, const char *f, int line) { (void)line; } -static void initOpensslLocks(void) { +static int initOpensslLocks(void) { unsigned ii, nlocks; if (CRYPTO_get_locking_callback() != NULL) { /* Someone already set the callback before us. Don't destroy it! */ - return; + return REDIS_OK; } nlocks = CRYPTO_num_locks(); - ossl_locks = malloc(sizeof(*ossl_locks) * nlocks); + ossl_locks = hi_malloc(sizeof(*ossl_locks) * nlocks); + if (ossl_locks == NULL) + return REDIS_ERR; + for (ii = 0; ii < nlocks; ii++) { sslLockInit(ossl_locks + ii); } CRYPTO_set_locking_callback(opensslDoLock); + return REDIS_OK; } #endif /* HIREDIS_USE_CRYPTO_LOCKS */ +int redisInitOpenSSL(void) +{ + SSL_library_init(); +#ifdef HIREDIS_USE_CRYPTO_LOCKS + initOpensslLocks(); +#endif + + return REDIS_OK; +} + +/** + * redisSSLContext helper context destruction. + */ + +const char *redisSSLContextGetError(redisSSLContextError error) +{ + switch (error) { + case REDIS_SSL_CTX_NONE: + return "No Error"; + case REDIS_SSL_CTX_CREATE_FAILED: + return "Failed to create OpenSSL SSL_CTX"; + case REDIS_SSL_CTX_CERT_KEY_REQUIRED: + return "Client cert and key must both be specified or skipped"; + case REDIS_SSL_CTX_CA_CERT_LOAD_FAILED: + return "Failed to load CA Certificate or CA Path"; + case REDIS_SSL_CTX_CLIENT_CERT_LOAD_FAILED: + return "Failed to load client certificate"; + case REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED: + return "Failed to load private key"; + default: + return "Unknown error code"; + } +} + +void redisFreeSSLContext(redisSSLContext *ctx) +{ + if (!ctx) + return; + + if (ctx->server_name) { + hi_free(ctx->server_name); + ctx->server_name = NULL; + } + + if (ctx->ssl_ctx) { + SSL_CTX_free(ctx->ssl_ctx); + ctx->ssl_ctx = NULL; + } + + hi_free(ctx); +} + + +/** + * redisSSLContext helper context initialization. + */ + +redisSSLContext *redisCreateSSLContext(const char *cacert_filename, const char *capath, + const char *cert_filename, const char *private_key_filename, + const char *server_name, redisSSLContextError *error) +{ + redisSSLContext *ctx = hi_calloc(1, sizeof(redisSSLContext)); + if (ctx == NULL) + goto error; + + ctx->ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + if (!ctx->ssl_ctx) { + if (error) *error = REDIS_SSL_CTX_CREATE_FAILED; + goto error; + } + + SSL_CTX_set_options(ctx->ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); + SSL_CTX_set_verify(ctx->ssl_ctx, SSL_VERIFY_PEER, NULL); + + if ((cert_filename != NULL && private_key_filename == NULL) || + (private_key_filename != NULL && cert_filename == NULL)) { + if (error) *error = REDIS_SSL_CTX_CERT_KEY_REQUIRED; + goto error; + } + + if (capath || cacert_filename) { + if (!SSL_CTX_load_verify_locations(ctx->ssl_ctx, cacert_filename, capath)) { + if (error) *error = REDIS_SSL_CTX_CA_CERT_LOAD_FAILED; + goto error; + } + } + + if (cert_filename) { + if (!SSL_CTX_use_certificate_chain_file(ctx->ssl_ctx, cert_filename)) { + if (error) *error = REDIS_SSL_CTX_CLIENT_CERT_LOAD_FAILED; + goto error; + } + if (!SSL_CTX_use_PrivateKey_file(ctx->ssl_ctx, private_key_filename, SSL_FILETYPE_PEM)) { + if (error) *error = REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED; + goto error; + } + } + + if (server_name) + ctx->server_name = hi_strdup(server_name); + + return ctx; + +error: + redisFreeSSLContext(ctx); + return NULL; +} + /** * SSL Connection initialization. */ -static int redisSSLConnect(redisContext *c, SSL_CTX *ssl_ctx, SSL *ssl) { - if (c->privdata) { + +static int redisSSLConnect(redisContext *c, SSL *ssl) { + if (c->privctx) { __redisSetError(c, REDIS_ERR_OTHER, "redisContext was already associated"); return REDIS_ERR; } - c->privdata = calloc(1, sizeof(redisSSLContext)); + + redisSSL *rssl = hi_calloc(1, sizeof(redisSSL)); + if (rssl == NULL) { + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; + } c->funcs = &redisContextSSLFuncs; - redisSSLContext *rssl = c->privdata; - - rssl->ssl_ctx = ssl_ctx; rssl->ssl = ssl; SSL_set_mode(rssl->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); @@ -183,12 +288,14 @@ static int redisSSLConnect(redisContext *c, SSL_CTX *ssl_ctx, SSL *ssl) { ERR_clear_error(); int rv = SSL_connect(rssl->ssl); if (rv == 1) { + c->privctx = rssl; return REDIS_OK; } rv = SSL_get_error(rssl->ssl, rv); if (((c->flags & REDIS_BLOCK) == 0) && (rv == SSL_ERROR_WANT_READ || rv == SSL_ERROR_WANT_WRITE)) { + c->privctx = rssl; return REDIS_OK; } @@ -203,83 +310,58 @@ static int redisSSLConnect(redisContext *c, SSL_CTX *ssl_ctx, SSL *ssl) { } __redisSetError(c, REDIS_ERR_IO, err); } + + hi_free(rssl); return REDIS_ERR; } +/** + * A wrapper around redisSSLConnect() for users who manage their own context and + * create their own SSL object. + */ + int redisInitiateSSL(redisContext *c, SSL *ssl) { - return redisSSLConnect(c, NULL, ssl); + return redisSSLConnect(c, ssl); } -int redisSecureConnection(redisContext *c, const char *capath, - const char *certpath, const char *keypath, const char *servername) { +/** + * A wrapper around redisSSLConnect() for users who use redisSSLContext and don't + * manage their own SSL objects. + */ - SSL_CTX *ssl_ctx = NULL; - SSL *ssl = NULL; +int redisInitiateSSLWithContext(redisContext *c, redisSSLContext *redis_ssl_ctx) +{ + if (!c || !redis_ssl_ctx) + return REDIS_ERR; - /* Initialize global OpenSSL stuff */ - static int isInit = 0; - if (!isInit) { - isInit = 1; - SSL_library_init(); -#ifdef HIREDIS_USE_CRYPTO_LOCKS - initOpensslLocks(); -#endif - } + /* We want to verify that redisSSLConnect() won't fail on this, as it will + * not own the SSL object in that case and we'll end up leaking. + */ + if (c->privctx) + return REDIS_ERR; - ssl_ctx = SSL_CTX_new(SSLv23_client_method()); - if (!ssl_ctx) { - __redisSetError(c, REDIS_ERR_OTHER, "Failed to create SSL_CTX"); - goto error; - } - -#ifdef HIREDIS_SSL_TRACE - SSL_CTX_set_info_callback(ssl_ctx, sslLogCallback); -#endif - SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); - SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); - if ((certpath != NULL && keypath == NULL) || (keypath != NULL && certpath == NULL)) { - __redisSetError(c, REDIS_ERR_OTHER, "certpath and keypath must be specified together"); - goto error; - } - - if (capath) { - if (!SSL_CTX_load_verify_locations(ssl_ctx, capath, NULL)) { - __redisSetError(c, REDIS_ERR_OTHER, "Invalid CA certificate"); - goto error; - } - } - if (certpath) { - if (!SSL_CTX_use_certificate_chain_file(ssl_ctx, certpath)) { - __redisSetError(c, REDIS_ERR_OTHER, "Invalid client certificate"); - goto error; - } - if (!SSL_CTX_use_PrivateKey_file(ssl_ctx, keypath, SSL_FILETYPE_PEM)) { - __redisSetError(c, REDIS_ERR_OTHER, "Invalid client key"); - goto error; - } - } - - ssl = SSL_new(ssl_ctx); + SSL *ssl = SSL_new(redis_ssl_ctx->ssl_ctx); if (!ssl) { __redisSetError(c, REDIS_ERR_OTHER, "Couldn't create new SSL instance"); goto error; } - if (servername) { - if (!SSL_set_tlsext_host_name(ssl, servername)) { - __redisSetError(c, REDIS_ERR_OTHER, "Couldn't set server name indication"); + + if (redis_ssl_ctx->server_name) { + if (!SSL_set_tlsext_host_name(ssl, redis_ssl_ctx->server_name)) { + __redisSetError(c, REDIS_ERR_OTHER, "Failed to set server_name/SNI"); goto error; } } - return redisSSLConnect(c, ssl_ctx, ssl); + return redisSSLConnect(c, ssl); error: - if (ssl) SSL_free(ssl); - if (ssl_ctx) SSL_CTX_free(ssl_ctx); + if (ssl) + SSL_free(ssl); return REDIS_ERR; } -static int maybeCheckWant(redisSSLContext *rssl, int rv) { +static int maybeCheckWant(redisSSL *rssl, int rv) { /** * If the error is WANT_READ or WANT_WRITE, the appropriate flags are set * and true is returned. False is returned otherwise @@ -299,23 +381,19 @@ static int maybeCheckWant(redisSSLContext *rssl, int rv) { * Implementation of redisContextFuncs for SSL connections. */ -static void redisSSLFreeContext(void *privdata){ - redisSSLContext *rsc = privdata; +static void redisSSLFree(void *privctx){ + redisSSL *rsc = privctx; if (!rsc) return; if (rsc->ssl) { SSL_free(rsc->ssl); rsc->ssl = NULL; } - if (rsc->ssl_ctx) { - SSL_CTX_free(rsc->ssl_ctx); - rsc->ssl_ctx = NULL; - } - free(rsc); + hi_free(rsc); } -static int redisSSLRead(redisContext *c, char *buf, size_t bufcap) { - redisSSLContext *rssl = c->privdata; +static ssize_t redisSSLRead(redisContext *c, char *buf, size_t bufcap) { + redisSSL *rssl = c->privctx; int nread = SSL_read(rssl->ssl, buf, bufcap); if (nread > 0) { @@ -356,10 +434,10 @@ static int redisSSLRead(redisContext *c, char *buf, size_t bufcap) { } } -static int redisSSLWrite(redisContext *c) { - redisSSLContext *rssl = c->privdata; +static ssize_t redisSSLWrite(redisContext *c) { + redisSSL *rssl = c->privctx; - size_t len = rssl->lastLen ? rssl->lastLen : sdslen(c->obuf); + size_t len = rssl->lastLen ? rssl->lastLen : hi_sdslen(c->obuf); int rv = SSL_write(rssl->ssl, c->obuf, len); if (rv > 0) { @@ -380,7 +458,7 @@ static int redisSSLWrite(redisContext *c) { static void redisSSLAsyncRead(redisAsyncContext *ac) { int rv; - redisSSLContext *rssl = ac->c.privdata; + redisSSL *rssl = ac->c.privctx; redisContext *c = &ac->c; rssl->wantRead = 0; @@ -410,7 +488,7 @@ static void redisSSLAsyncRead(redisAsyncContext *ac) { static void redisSSLAsyncWrite(redisAsyncContext *ac) { int rv, done = 0; - redisSSLContext *rssl = ac->c.privdata; + redisSSL *rssl = ac->c.privctx; redisContext *c = &ac->c; rssl->pendingWrite = 0; @@ -439,7 +517,7 @@ static void redisSSLAsyncWrite(redisAsyncContext *ac) { } redisContextFuncs redisContextSSLFuncs = { - .free_privdata = redisSSLFreeContext, + .free_privctx = redisSSLFree, .async_read = redisSSLAsyncRead, .async_write = redisSSLAsyncWrite, .read = redisSSLRead, diff --git a/deps/hiredis/test.c b/deps/hiredis/test.c index 8668e1856..829536739 100644 --- a/deps/hiredis/test.c +++ b/deps/hiredis/test.c @@ -1,22 +1,24 @@ #include "fmacros.h" +#include "sockcompat.h" #include #include #include +#ifndef _WIN32 #include -#include #include -#include +#endif #include -#include #include #include #include #include "hiredis.h" +#include "async.h" #ifdef HIREDIS_TEST_SSL #include "hiredis_ssl.h" #endif #include "net.h" +#include "win32.h" enum connection_type { CONN_TCP, @@ -47,15 +49,35 @@ struct config { } ssl; }; +struct privdata { + int dtor_counter; +}; + +struct pushCounters { + int nil; + int str; +}; + +#ifdef HIREDIS_TEST_SSL +redisSSLContext *_ssl_ctx = NULL; +#endif + /* The following lines make up our testing "framework" :) */ -static int tests = 0, fails = 0; +static int tests = 0, fails = 0, skips = 0; #define test(_s) { printf("#%02d ", ++tests); printf(_s); } #define test_cond(_c) if(_c) printf("\033[0;32mPASSED\033[0;0m\n"); else {printf("\033[0;31mFAILED\033[0;0m\n"); fails++;} +#define test_skipped() { printf("\033[01;33mSKIPPED\033[0;0m\n"); skips++; } static long long usec(void) { +#ifndef _MSC_VER struct timeval tv; gettimeofday(&tv,NULL); return (((long long)tv.tv_sec)*1000000)+tv.tv_usec; +#else + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + return (((long long)ft.dwHighDateTime << 32) | ft.dwLowDateTime) / 10; +#endif } /* The assert() calls below have side effects, so we need assert() @@ -65,6 +87,43 @@ static long long usec(void) { #define assert(e) (void)(e) #endif +/* Helper to extract Redis version information. Aborts on any failure. */ +#define REDIS_VERSION_FIELD "redis_version:" +void get_redis_version(redisContext *c, int *majorptr, int *minorptr) { + redisReply *reply; + char *eptr, *s, *e; + int major, minor; + + reply = redisCommand(c, "INFO"); + if (reply == NULL || c->err || reply->type != REDIS_REPLY_STRING) + goto abort; + if ((s = strstr(reply->str, REDIS_VERSION_FIELD)) == NULL) + goto abort; + + s += strlen(REDIS_VERSION_FIELD); + + /* We need a field terminator and at least 'x.y.z' (5) bytes of data */ + if ((e = strstr(s, "\r\n")) == NULL || (e - s) < 5) + goto abort; + + /* Extract version info */ + major = strtol(s, &eptr, 10); + if (*eptr != '.') goto abort; + minor = strtol(eptr+1, NULL, 10); + + /* Push info the caller wants */ + if (majorptr) *majorptr = major; + if (minorptr) *minorptr = minor; + + freeReplyObject(reply); + return; + +abort: + freeReplyObject(reply); + fprintf(stderr, "Error: Cannot determine Redis version, aborting\n"); + exit(1); +} + static redisContext *select_database(redisContext *c) { redisReply *reply; @@ -87,6 +146,26 @@ static redisContext *select_database(redisContext *c) { return c; } +/* Switch protocol */ +static void send_hello(redisContext *c, int version) { + redisReply *reply; + int expected; + + reply = redisCommand(c, "HELLO %d", version); + expected = version == 3 ? REDIS_REPLY_MAP : REDIS_REPLY_ARRAY; + assert(reply != NULL && reply->type == expected); + freeReplyObject(reply); +} + +/* Togggle client tracking */ +static void send_client_tracking(redisContext *c, const char *str) { + redisReply *reply; + + reply = redisCommand(c, "CLIENT TRACKING %s", str); + assert(reply != NULL && reply->type == REDIS_REPLY_STATUS); + freeReplyObject(reply); +} + static int disconnect(redisContext *c, int keep_fd) { redisReply *reply; @@ -105,9 +184,9 @@ static int disconnect(redisContext *c, int keep_fd) { return -1; } -static void do_ssl_handshake(redisContext *c, struct config config) { +static void do_ssl_handshake(redisContext *c) { #ifdef HIREDIS_TEST_SSL - redisSecureConnection(c, config.ssl.ca_cert, config.ssl.cert, config.ssl.key, NULL); + redisInitiateSSLWithContext(c, _ssl_ctx); if (c->err) { printf("SSL error: %s\n", c->errstr); redisFree(c); @@ -115,7 +194,6 @@ static void do_ssl_handshake(redisContext *c, struct config config) { } #else (void) c; - (void) config; #endif } @@ -150,7 +228,7 @@ static redisContext *do_connect(struct config config) { } if (config.type == CONN_SSL) { - do_ssl_handshake(c, config); + do_ssl_handshake(c); } return select_database(c); @@ -160,7 +238,7 @@ static void do_reconnect(redisContext *c, struct config config) { redisReconnect(c); if (config.type == CONN_SSL) { - do_ssl_handshake(c, config); + do_ssl_handshake(c); } } @@ -172,43 +250,43 @@ static void test_format_commands(void) { len = redisFormatCommand(&cmd,"SET foo bar"); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command with %%s string interpolation: "); len = redisFormatCommand(&cmd,"SET %s %s","foo","bar"); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command with %%s and an empty string: "); len = redisFormatCommand(&cmd,"SET %s %s","foo",""); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(0+2)); - free(cmd); + hi_free(cmd); test("Format command with an empty string in between proper interpolations: "); len = redisFormatCommand(&cmd,"SET %s %s","","foo"); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$0\r\n\r\n$3\r\nfoo\r\n",len) == 0 && len == 4+4+(3+2)+4+(0+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command with %%b string interpolation: "); len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"b\0r",(size_t)3); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nb\0r\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command with %%b and an empty string: "); len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"",(size_t)0); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(0+2)); - free(cmd); + hi_free(cmd); test("Format command with literal %%: "); len = redisFormatCommand(&cmd,"SET %% %%"); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$1\r\n%\r\n$1\r\n%\r\n",len) == 0 && len == 4+4+(3+2)+4+(1+2)+4+(1+2)); - free(cmd); + hi_free(cmd); /* Vararg width depends on the type. These tests make sure that the * width is correctly determined using the format and subsequent varargs @@ -219,7 +297,7 @@ static void test_format_commands(void) { len = redisFormatCommand(&cmd,"key:%08" fmt " str:%s", value, "hello"); \ test_cond(strncmp(cmd,"*2\r\n$12\r\nkey:00000123\r\n$9\r\nstr:hello\r\n",len) == 0 && \ len == 4+5+(12+2)+4+(9+2)); \ - free(cmd); \ + hi_free(cmd); \ } while(0) #define FLOAT_WIDTH_TEST(type) do { \ @@ -228,7 +306,7 @@ static void test_format_commands(void) { len = redisFormatCommand(&cmd,"key:%08.3f str:%s", value, "hello"); \ test_cond(strncmp(cmd,"*2\r\n$12\r\nkey:0123.000\r\n$9\r\nstr:hello\r\n",len) == 0 && \ len == 4+5+(12+2)+4+(9+2)); \ - free(cmd); \ + hi_free(cmd); \ } while(0) INTEGER_WIDTH_TEST("d", int); @@ -259,29 +337,29 @@ static void test_format_commands(void) { len = redisFormatCommandArgv(&cmd,argc,argv,NULL); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command by passing argc/argv with lengths: "); len = redisFormatCommandArgv(&cmd,argc,argv,lens); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(7+2)+4+(3+2)); - free(cmd); + hi_free(cmd); - sds sds_cmd; + hisds sds_cmd; - sds_cmd = sdsempty(); - test("Format command into sds by passing argc/argv without lengths: "); + sds_cmd = NULL; + test("Format command into hisds by passing argc/argv without lengths: "); len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,NULL); test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - sdsfree(sds_cmd); + hi_sdsfree(sds_cmd); - sds_cmd = sdsempty(); - test("Format command into sds by passing argc/argv with lengths: "); + sds_cmd = NULL; + test("Format command into hisds by passing argc/argv with lengths: "); len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,lens); test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(7+2)+4+(3+2)); - sdsfree(sds_cmd); + hi_sdsfree(sds_cmd); } static void test_append_formatted_commands(struct config config) { @@ -300,7 +378,7 @@ static void test_append_formatted_commands(struct config config) { assert(redisGetReply(c, (void*)&reply) == REDIS_OK); - free(cmd); + hi_free(cmd); freeReplyObject(reply); disconnect(c, 0); @@ -308,7 +386,7 @@ static void test_append_formatted_commands(struct config config) { static void test_reply_reader(void) { redisReader *reader; - void *reply; + void *reply, *root; int ret; int i; @@ -332,16 +410,26 @@ static void test_reply_reader(void) { strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0); redisReaderFree(reader); - test("Set error on nested multi bulks with depth > 7: "); reader = redisReaderCreate(); - - for (i = 0; i < 9; i++) { - redisReaderFeed(reader,(char*)"*1\r\n",4); + test("Can handle arbitrarily nested multi-bulks: "); + for (i = 0; i < 128; i++) { + redisReaderFeed(reader,(char*)"*1\r\n", 4); } + redisReaderFeed(reader,(char*)"$6\r\nLOLWUT\r\n",12); + ret = redisReaderGetReply(reader,&reply); + root = reply; /* Keep track of the root reply */ + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_ARRAY && + ((redisReply*)reply)->elements == 1); - ret = redisReaderGetReply(reader,NULL); - test_cond(ret == REDIS_ERR && - strncasecmp(reader->errstr,"No support for",14) == 0); + test("Can parse arbitrarily nested multi-bulks correctly: "); + while(i--) { + assert(reply != NULL && ((redisReply*)reply)->type == REDIS_REPLY_ARRAY); + reply = ((redisReply*)reply)->element[0]; + } + test_cond(((redisReply*)reply)->type == REDIS_REPLY_STRING && + !memcmp(((redisReply*)reply)->str, "LOLWUT", 6)); + freeReplyObject(root); redisReaderFree(reader); test("Correctly parses LLONG_MAX: "); @@ -400,6 +488,16 @@ static void test_reply_reader(void) { freeReplyObject(reply); redisReaderFree(reader); + test("Can configure maximum multi-bulk elements: "); + reader = redisReaderCreate(); + reader->maxelements = 1024; + redisReaderFeed(reader, "*1025\r\n", 7); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr, "Multi-bulk length out of range") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + #if LLONG_MAX > SIZE_MAX test("Set error when array > SIZE_MAX: "); reader = redisReaderCreate(); @@ -459,6 +557,32 @@ static void test_reply_reader(void) { ((redisReply*)reply)->elements == 0); freeReplyObject(reply); redisReaderFree(reader); + + /* RESP3 verbatim strings (GitHub issue #802) */ + test("Can parse RESP3 verbatim strings: "); + reader = redisReaderCreate(); + redisReaderFeed(reader,(char*)"=10\r\ntxt:LOLWUT\r\n",17); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_VERB && + !memcmp(((redisReply*)reply)->str,"LOLWUT", 6)); + freeReplyObject(reply); + redisReaderFree(reader); + + /* RESP3 push messages (Github issue #815) */ + test("Can parse RESP3 push messages: "); + reader = redisReaderCreate(); + redisReaderFeed(reader,(char*)">2\r\n$6\r\nLOLWUT\r\n:42\r\n",21); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_PUSH && + ((redisReply*)reply)->elements == 2 && + ((redisReply*)reply)->element[0]->type == REDIS_REPLY_STRING && + !memcmp(((redisReply*)reply)->element[0]->str,"LOLWUT",6) && + ((redisReply*)reply)->element[1]->type == REDIS_REPLY_INTEGER && + ((redisReply*)reply)->element[1]->integer == 42); + freeReplyObject(reply); + redisReaderFree(reader); } static void test_free_null(void) { @@ -474,6 +598,47 @@ static void test_free_null(void) { test_cond(reply == NULL); } +static void *hi_malloc_fail(size_t size) { + (void)size; + return NULL; +} + +static void *hi_calloc_fail(size_t nmemb, size_t size) { + (void)nmemb; + (void)size; + return NULL; +} + +static void *hi_realloc_fail(void *ptr, size_t size) { + (void)ptr; + (void)size; + return NULL; +} + +static void test_allocator_injection(void) { + hiredisAllocFuncs ha = { + .mallocFn = hi_malloc_fail, + .callocFn = hi_calloc_fail, + .reallocFn = hi_realloc_fail, + .strdupFn = strdup, + .freeFn = free, + }; + + // Override hiredis allocators + hiredisSetAllocators(&ha); + + test("redisContext uses injected allocators: "); + redisContext *c = redisConnect("localhost", 6379); + test_cond(c == NULL); + + test("redisReader uses injected allocators: "); + redisReader *reader = redisReaderCreate(); + test_cond(reader == NULL); + + // Return allocators to default + hiredisResetAllocators(); +} + #define HIREDIS_BAD_DOMAIN "idontexist-noreally.com" static void test_blocking_connection_errors(void) { redisContext *c; @@ -491,19 +656,19 @@ static void test_blocking_connection_errors(void) { (strcmp(c->errstr, "Name or service not known") == 0 || strcmp(c->errstr, "Can't resolve: " HIREDIS_BAD_DOMAIN) == 0 || strcmp(c->errstr, "Name does not resolve") == 0 || - strcmp(c->errstr, - "nodename nor servname provided, or not known") == 0 || + strcmp(c->errstr, "nodename nor servname provided, or not known") == 0 || strcmp(c->errstr, "No address associated with hostname") == 0 || strcmp(c->errstr, "Temporary failure in name resolution") == 0 || - strcmp(c->errstr, - "hostname nor servname provided, or not known") == 0 || - strcmp(c->errstr, "no address associated with name") == 0)); + strcmp(c->errstr, "hostname nor servname provided, or not known") == 0 || + strcmp(c->errstr, "no address associated with name") == 0 || + strcmp(c->errstr, "No such host is known. ") == 0)); redisFree(c); } else { printf("Skipping NXDOMAIN test. Found evil ISP!\n"); freeaddrinfo(ai_tmp); } +#ifndef _WIN32 test("Returns error when the port is not open: "); c = redisConnect((char*)"localhost", 1); test_cond(c->err == REDIS_ERR_IO && @@ -514,11 +679,166 @@ static void test_blocking_connection_errors(void) { c = redisConnectUnix((char*)"/tmp/idontexist.sock"); test_cond(c->err == REDIS_ERR_IO); /* Don't care about the message... */ redisFree(c); +#endif +} + +/* Test push handler */ +void push_handler(void *privdata, void *r) { + struct pushCounters *pcounts = privdata; + redisReply *reply = r, *payload; + + assert(reply && reply->type == REDIS_REPLY_PUSH && reply->elements == 2); + + payload = reply->element[1]; + if (payload->type == REDIS_REPLY_ARRAY) { + payload = payload->element[0]; + } + + if (payload->type == REDIS_REPLY_STRING) { + pcounts->str++; + } else if (payload->type == REDIS_REPLY_NIL) { + pcounts->nil++; + } + + freeReplyObject(reply); +} + +/* Dummy function just to test setting a callback with redisOptions */ +void push_handler_async(redisAsyncContext *ac, void *reply) { + (void)ac; + (void)reply; +} + +static void test_resp3_push_handler(redisContext *c) { + struct pushCounters pc = {0}; + redisPushFn *old = NULL; + redisReply *reply; + void *privdata; + + /* Switch to RESP3 and turn on client tracking */ + send_hello(c, 3); + send_client_tracking(c, "ON"); + privdata = c->privdata; + c->privdata = &pc; + + reply = redisCommand(c, "GET key:0"); + assert(reply != NULL); + freeReplyObject(reply); + + test("RESP3 PUSH messages are handled out of band by default: "); + reply = redisCommand(c, "SET key:0 val:0"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS); + freeReplyObject(reply); + + assert((reply = redisCommand(c, "GET key:0")) != NULL); + freeReplyObject(reply); + + old = redisSetPushCallback(c, push_handler); + test("We can set a custom RESP3 PUSH handler: "); + reply = redisCommand(c, "SET key:0 val:0"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && pc.str == 1); + freeReplyObject(reply); + + test("We properly handle a NIL invalidation payload: "); + reply = redisCommand(c, "FLUSHDB"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && pc.nil == 1); + freeReplyObject(reply); + + /* Unset the push callback and generate an invalidate message making + * sure it is not handled out of band. */ + test("With no handler, PUSH replies come in-band: "); + redisSetPushCallback(c, NULL); + assert((reply = redisCommand(c, "GET key:0")) != NULL); + freeReplyObject(reply); + assert((reply = redisCommand(c, "SET key:0 invalid")) != NULL); + test_cond(reply->type == REDIS_REPLY_PUSH); + freeReplyObject(reply); + + test("With no PUSH handler, no replies are lost: "); + assert(redisGetReply(c, (void**)&reply) == REDIS_OK); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS); + freeReplyObject(reply); + + /* Return to the originally set PUSH handler */ + assert(old != NULL); + redisSetPushCallback(c, old); + + /* Switch back to RESP2 and disable tracking */ + c->privdata = privdata; + send_client_tracking(c, "OFF"); + send_hello(c, 2); +} + +redisOptions get_redis_tcp_options(struct config config) { + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, config.tcp.host, config.tcp.port); + return options; +} + +static void test_resp3_push_options(struct config config) { + redisAsyncContext *ac; + redisContext *c; + redisOptions options; + + test("We set a default RESP3 handler for redisContext: "); + options = get_redis_tcp_options(config); + assert((c = redisConnectWithOptions(&options)) != NULL); + test_cond(c->push_cb != NULL); + redisFree(c); + + test("We don't set a default RESP3 push handler for redisAsyncContext: "); + options = get_redis_tcp_options(config); + assert((ac = redisAsyncConnectWithOptions(&options)) != NULL); + test_cond(ac->c.push_cb == NULL); + redisAsyncFree(ac); + + test("Our REDIS_OPT_NO_PUSH_AUTOFREE flag works: "); + options = get_redis_tcp_options(config); + options.options |= REDIS_OPT_NO_PUSH_AUTOFREE; + assert((c = redisConnectWithOptions(&options)) != NULL); + test_cond(c->push_cb == NULL); + redisFree(c); + + test("We can use redisOptions to set a custom PUSH handler for redisContext: "); + options = get_redis_tcp_options(config); + options.push_cb = push_handler; + assert((c = redisConnectWithOptions(&options)) != NULL); + test_cond(c->push_cb == push_handler); + redisFree(c); + + test("We can use redisOptions to set a custom PUSH handler for redisAsyncContext: "); + options = get_redis_tcp_options(config); + options.async_push_cb = push_handler_async; + assert((ac = redisAsyncConnectWithOptions(&options)) != NULL); + test_cond(ac->push_cb == push_handler_async); + redisAsyncFree(ac); +} + +void free_privdata(void *privdata) { + struct privdata *data = privdata; + data->dtor_counter++; +} + +static void test_privdata_hooks(struct config config) { + struct privdata data = {0}; + redisOptions options; + redisContext *c; + + test("We can use redisOptions to set privdata: "); + options = get_redis_tcp_options(config); + REDIS_OPTIONS_SET_PRIVDATA(&options, &data, free_privdata); + assert((c = redisConnectWithOptions(&options)) != NULL); + test_cond(c->privdata == &data); + + test("Our privdata destructor fires when we free the context: "); + redisFree(c); + test_cond(data.dtor_counter == 1); } static void test_blocking_connection(struct config config) { redisContext *c; redisReply *reply; + int major; c = do_connect(config); @@ -591,14 +911,42 @@ static void test_blocking_connection(struct config config) { strcasecmp(reply->element[1]->str,"pong") == 0); freeReplyObject(reply); + /* Make sure passing NULL to redisGetReply is safe */ + test("Can pass NULL to redisGetReply: "); + assert(redisAppendCommand(c, "PING") == REDIS_OK); + test_cond(redisGetReply(c, NULL) == REDIS_OK); + + get_redis_version(c, &major, NULL); + if (major >= 6) test_resp3_push_handler(c); + test_resp3_push_options(config); + + test_privdata_hooks(config); + disconnect(c, 0); } +/* Send DEBUG SLEEP 0 to detect if we have this command */ +static int detect_debug_sleep(redisContext *c) { + int detected; + redisReply *reply = redisCommand(c, "DEBUG SLEEP 0\r\n"); + + if (reply == NULL || c->err) { + const char *cause = c->err ? c->errstr : "(none)"; + fprintf(stderr, "Error testing for DEBUG SLEEP (Redis error: %s), exiting\n", cause); + exit(-1); + } + + detected = reply->type == REDIS_REPLY_STATUS; + freeReplyObject(reply); + + return detected; +} + static void test_blocking_connection_timeouts(struct config config) { redisContext *c; redisReply *reply; ssize_t s; - const char *cmd = "DEBUG SLEEP 3\r\n"; + const char *sleep_cmd = "DEBUG SLEEP 3\r\n"; struct timeval tv; c = do_connect(config); @@ -615,14 +963,24 @@ static void test_blocking_connection_timeouts(struct config config) { c = do_connect(config); test("Does not return a reply when the command times out: "); - redisAppendFormattedCommand(c, cmd, strlen(cmd)); - s = c->funcs->write(c); - tv.tv_sec = 0; - tv.tv_usec = 10000; - redisSetTimeout(c, tv); - reply = redisCommand(c, "GET foo"); - test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_IO && strcmp(c->errstr, "Resource temporarily unavailable") == 0); - freeReplyObject(reply); + if (detect_debug_sleep(c)) { + redisAppendFormattedCommand(c, sleep_cmd, strlen(sleep_cmd)); + s = c->funcs->write(c); + tv.tv_sec = 0; + tv.tv_usec = 10000; + redisSetTimeout(c, tv); + reply = redisCommand(c, "GET foo"); +#ifndef _WIN32 + test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_IO && + strcmp(c->errstr, "Resource temporarily unavailable") == 0); +#else + test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_TIMEOUT && + strcmp(c->errstr, "recv timeout") == 0); +#endif + freeReplyObject(reply); + } else { + test_skipped(); + } test("Reconnect properly reconnects after a timeout: "); do_reconnect(c, config); @@ -649,18 +1007,7 @@ static void test_blocking_io_errors(struct config config) { /* Connect to target given by config. */ c = do_connect(config); - { - /* Find out Redis version to determine the path for the next test */ - const char *field = "redis_version:"; - char *p, *eptr; - - reply = redisCommand(c,"INFO"); - p = strstr(reply->str,field); - major = strtol(p+strlen(field),&eptr,10); - p = eptr+1; /* char next to the first "." */ - minor = strtol(p,&eptr,10); - freeReplyObject(reply); - } + get_redis_version(c, &major, &minor); test("Returns I/O error when the connection is lost: "); reply = redisCommand(c,"QUIT"); @@ -674,6 +1021,7 @@ static void test_blocking_io_errors(struct config config) { test_cond(reply == NULL); } +#ifndef _WIN32 /* On 2.0, QUIT will cause the connection to be closed immediately and * the read(2) for the reply on QUIT will set the error to EOF. * On >2.0, QUIT will return with OK and another read(2) needed to be @@ -681,14 +1029,19 @@ static void test_blocking_io_errors(struct config config) { * conditions, the error will be set to EOF. */ assert(c->err == REDIS_ERR_EOF && strcmp(c->errstr,"Server closed the connection") == 0); +#endif redisFree(c); c = do_connect(config); test("Returns I/O error on socket timeout: "); struct timeval tv = { 0, 1000 }; assert(redisSetTimeout(c,tv) == REDIS_OK); - test_cond(redisGetReply(c,&_reply) == REDIS_ERR && - c->err == REDIS_ERR_IO && errno == EAGAIN); + int respcode = redisGetReply(c,&_reply); +#ifndef _WIN32 + test_cond(respcode == REDIS_ERR && c->err == REDIS_ERR_IO && errno == EAGAIN); +#else + test_cond(respcode == REDIS_ERR && c->err == REDIS_ERR_TIMEOUT); +#endif redisFree(c); } @@ -716,6 +1069,18 @@ static void test_invalid_timeout_errors(struct config config) { redisFree(c); } +/* Wrap malloc to abort on failure so OOM checks don't make the test logic + * harder to follow. */ +void *hi_malloc_safe(size_t size) { + void *ptr = hi_malloc(size); + if (ptr == NULL) { + fprintf(stderr, "Error: Out of memory\n"); + exit(-1); + } + + return ptr; +} + static void test_throughput(struct config config) { redisContext *c = do_connect(config); redisReply **replies; @@ -727,7 +1092,7 @@ static void test_throughput(struct config config) { freeReplyObject(redisCommand(c,"LPUSH mylist foo")); num = 1000; - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); t1 = usec(); for (i = 0; i < num; i++) { replies[i] = redisCommand(c,"PING"); @@ -735,10 +1100,10 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx PING: %.3fs)\n", num, (t2-t1)/1000000.0); - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); t1 = usec(); for (i = 0; i < num; i++) { replies[i] = redisCommand(c,"LRANGE mylist 0 499"); @@ -747,10 +1112,10 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx LRANGE with 500 elements: %.3fs)\n", num, (t2-t1)/1000000.0); - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); t1 = usec(); for (i = 0; i < num; i++) { replies[i] = redisCommand(c, "INCRBY incrkey %d", 1000000); @@ -758,11 +1123,11 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx INCRBY: %.3fs)\n", num, (t2-t1)/1000000.0); num = 10000; - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); for (i = 0; i < num; i++) redisAppendCommand(c,"PING"); t1 = usec(); @@ -772,10 +1137,10 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx PING (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); for (i = 0; i < num; i++) redisAppendCommand(c,"LRANGE mylist 0 499"); t1 = usec(); @@ -786,10 +1151,10 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx LRANGE with 500 elements (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); for (i = 0; i < num; i++) redisAppendCommand(c,"INCRBY incrkey %d", 1000000); t1 = usec(); @@ -799,7 +1164,7 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx INCRBY (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); disconnect(c, 0); @@ -916,9 +1281,8 @@ int main(int argc, char **argv) { }; int throughput = 1; int test_inherit_fd = 1; - - /* Ignore broken pipe signal (for I/O error tests). */ - signal(SIGPIPE, SIG_IGN); + int skips_as_fails = 0; + int test_unix_socket; /* Parse command line options. */ argv++; argc--; @@ -936,6 +1300,8 @@ int main(int argc, char **argv) { throughput = 0; } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) { test_inherit_fd = 0; + } else if (argc >= 1 && !strcmp(argv[0],"--skips-as-fails")) { + skips_as_fails = 1; #ifdef HIREDIS_TEST_SSL } else if (argc >= 2 && !strcmp(argv[0],"--ssl-port")) { argv++; argc--; @@ -960,6 +1326,19 @@ int main(int argc, char **argv) { argv++; argc--; } +#ifndef _WIN32 + /* Ignore broken pipe signal (for I/O error tests). */ + signal(SIGPIPE, SIG_IGN); + + test_unix_socket = access(cfg.unix_sock.path, F_OK) == 0; + +#else + /* Unix sockets don't exist in Windows */ + test_unix_socket = 0; +#endif + + test_allocator_injection(); + test_format_commands(); test_reply_reader(); test_blocking_connection_errors(); @@ -974,15 +1353,25 @@ int main(int argc, char **argv) { test_append_formatted_commands(cfg); if (throughput) test_throughput(cfg); - printf("\nTesting against Unix socket connection (%s):\n", cfg.unix_sock.path); - cfg.type = CONN_UNIX; - test_blocking_connection(cfg); - test_blocking_connection_timeouts(cfg); - test_blocking_io_errors(cfg); - if (throughput) test_throughput(cfg); + printf("\nTesting against Unix socket connection (%s): ", cfg.unix_sock.path); + if (test_unix_socket) { + printf("\n"); + cfg.type = CONN_UNIX; + test_blocking_connection(cfg); + test_blocking_connection_timeouts(cfg); + test_blocking_io_errors(cfg); + if (throughput) test_throughput(cfg); + } else { + test_skipped(); + } #ifdef HIREDIS_TEST_SSL if (cfg.ssl.port && cfg.ssl.host) { + + redisInitOpenSSL(); + _ssl_ctx = redisCreateSSLContext(cfg.ssl.ca_cert, NULL, cfg.ssl.cert, cfg.ssl.key, NULL, NULL); + assert(_ssl_ctx != NULL); + printf("\nTesting against SSL connection (%s:%d):\n", cfg.ssl.host, cfg.ssl.port); cfg.type = CONN_SSL; @@ -992,21 +1381,31 @@ int main(int argc, char **argv) { test_invalid_timeout_errors(cfg); test_append_formatted_commands(cfg); if (throughput) test_throughput(cfg); + + redisFreeSSLContext(_ssl_ctx); + _ssl_ctx = NULL; } #endif if (test_inherit_fd) { - printf("\nTesting against inherited fd (%s):\n", cfg.unix_sock.path); - cfg.type = CONN_FD; - test_blocking_connection(cfg); + printf("\nTesting against inherited fd (%s): ", cfg.unix_sock.path); + if (test_unix_socket) { + printf("\n"); + cfg.type = CONN_FD; + test_blocking_connection(cfg); + } else { + test_skipped(); + } } - - if (fails) { + if (fails || (skips_as_fails && skips)) { printf("*** %d TESTS FAILED ***\n", fails); + if (skips) { + printf("*** %d TESTS SKIPPED ***\n", skips); + } return 1; } - printf("ALL TESTS PASSED\n"); + printf("ALL TESTS PASSED (%d skipped)\n", skips); return 0; } diff --git a/deps/hiredis/test.sh b/deps/hiredis/test.sh index 2cab9e6fb..c72bcb0dc 100755 --- a/deps/hiredis/test.sh +++ b/deps/hiredis/test.sh @@ -4,7 +4,9 @@ REDIS_SERVER=${REDIS_SERVER:-redis-server} REDIS_PORT=${REDIS_PORT:-56379} REDIS_SSL_PORT=${REDIS_SSL_PORT:-56443} TEST_SSL=${TEST_SSL:-0} +SKIPS_AS_FAILS=${SKIPS_AS_FAILS-:0} SSL_TEST_ARGS= +SKIPS_ARG= tmpdir=$(mktemp -d) PID_FILE=${tmpdir}/hiredis-test-redis.pid @@ -67,4 +69,10 @@ fi cat ${tmpdir}/redis.conf ${REDIS_SERVER} ${tmpdir}/redis.conf -${TEST_PREFIX:-} ./hiredis-test -h 127.0.0.1 -p ${REDIS_PORT} -s ${SOCK_FILE} ${SSL_TEST_ARGS} +# Wait until we detect the unix socket +while [ ! -S "${SOCK_FILE}" ]; do sleep 1; done + +# Treat skips as failures if directed +[ "$SKIPS_AS_FAILS" = 1 ] && SKIPS_ARG="--skips-as-fails" + +${TEST_PREFIX:-} ./hiredis-test -h 127.0.0.1 -p ${REDIS_PORT} -s ${SOCK_FILE} ${SSL_TEST_ARGS} ${SKIPS_ARG} diff --git a/deps/jemalloc/configure.ac b/deps/jemalloc/configure.ac index 261d81c00..f867172f7 100644 --- a/deps/jemalloc/configure.ac +++ b/deps/jemalloc/configure.ac @@ -517,7 +517,7 @@ CTARGET='-o $@' LDTARGET='-o $@' TEST_LD_MODE= EXTRA_LDFLAGS= -ARFLAGS='crus' +ARFLAGS='crs' AROUT=' $@' CC_MM=1 diff --git a/deps/lua/src/lauxlib.c b/deps/lua/src/lauxlib.c index 10f14e2c0..751f1e887 100644 --- a/deps/lua/src/lauxlib.c +++ b/deps/lua/src/lauxlib.c @@ -575,7 +575,7 @@ LUALIB_API int luaL_loadfile (lua_State *L, const char *filename) { if (lf.f == NULL) return errfile(L, "reopen", fnameindex); /* skip eventual `#!...' */ while ((c = getc(lf.f)) != EOF && c != LUA_SIGNATURE[0]) ; - lf.extraline = 0; + lf.extraline = 0; } ungetc(c, lf.f); status = lua_load(L, getF, &lf, lua_tostring(L, -1)); diff --git a/deps/lua/src/ldo.c b/deps/lua/src/ldo.c index 939940a4c..e641dfcd9 100644 --- a/deps/lua/src/ldo.c +++ b/deps/lua/src/ldo.c @@ -493,7 +493,7 @@ static void f_parser (lua_State *L, void *ud) { Proto *tf; Closure *cl; struct SParser *p = cast(struct SParser *, ud); - int c = luaZ_lookahead(p->z); + luaZ_lookahead(p->z); luaC_checkGC(L); tf = (luaY_parser)(L, p->z, &p->buff, p->name); diff --git a/deps/lua/src/ltablib.c b/deps/lua/src/ltablib.c index b6d9cb4ac..0bdac7f6c 100644 --- a/deps/lua/src/ltablib.c +++ b/deps/lua/src/ltablib.c @@ -137,7 +137,7 @@ static void addfield (lua_State *L, luaL_Buffer *b, int i) { if (!lua_isstring(L, -1)) luaL_error(L, "invalid value (%s) at index %d in table for " LUA_QL("concat"), luaL_typename(L, -1), i); - luaL_addvalue(b); + luaL_addvalue(b); } diff --git a/pkg/deb/conf/keydb.conf b/pkg/deb/conf/keydb.conf index 9cf1773c7..22844de5a 100644 --- a/pkg/deb/conf/keydb.conf +++ b/pkg/deb/conf/keydb.conf @@ -3,7 +3,7 @@ # Note that in order to read the configuration file, Redis must be # started with the file path as first argument: # -# ./keydb-server /path/to/redis.conf +# ./keydb-server /path/to/keydb.conf # Note on units: when memory size is needed, it is possible to specify # it in the usual form of 1k 5GB 4M and so forth: @@ -24,7 +24,7 @@ # to customize a few per-server settings. Include files can include # other files, so use this wisely. # -# Notice option "include" won't be rewritten by command "CONFIG REWRITE" +# Note that option "include" won't be rewritten by command "CONFIG REWRITE" # from admin or Redis Sentinel. Since Redis always uses the last processed # line as value of a configuration directive, you'd better put includes # at the beginning of this file to avoid overwriting config change at runtime. @@ -46,25 +46,31 @@ ################################## NETWORK ##################################### # By default, if no "bind" configuration directive is specified, Redis listens -# for connections from all the network interfaces available on the server. +# for connections from all available network interfaces on the host machine. # It is possible to listen to just one or multiple selected interfaces using # the "bind" configuration directive, followed by one or more IP addresses. +# Each address can be prefixed by "-", which means that KeyDB will not fail to +# start if the address is not available. Being not available only refers to +# addresses that does not correspond to any network interfece. Addresses that +# are already in use will always fail, and unsupported protocols will always BE +# silently skipped. # # Examples: # -# bind 192.168.1.100 10.0.0.1 -# bind 127.0.0.1 ::1 +# bind 192.168.1.100 10.0.0.1 # listens on two specific IPv4 addresses +# bind 127.0.0.1 ::1 # listens on loopback IPv4 and IPv6 +# bind * -::* # like the default, all available interfaces # # ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the # internet, binding to all the interfaces is dangerous and will expose the # instance to everybody on the internet. So by default we uncomment the -# following bind directive, that will force Redis to listen only into -# the IPv4 loopback interface address (this means Redis will be able to -# accept connections only from clients running into the same computer it -# is running). +# following bind directive, that will force Redis to listen only on the +# IPv4 and IPv6 (if available) loopback interface addresses (this means Redis +# will only be able to accept client connections from the same host that it is +# running on). # # IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES -# JUST COMMENT THE FOLLOWING LINE. +# JUST COMMENT OUT THE FOLLOWING LINE. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ bind 127.0.0.1 ::1 @@ -93,8 +99,8 @@ port 6379 # TCP listen() backlog. # -# In high requests-per-second environments you need an high backlog in order -# to avoid slow clients connections issues. Note that the Linux kernel +# In high requests-per-second environments you need a high backlog in order +# to avoid slow clients connection issues. Note that the Linux kernel # will silently truncate it to the value of /proc/sys/net/core/somaxconn so # make sure to raise both the value of somaxconn and tcp_max_syn_backlog # in order to get the desired effect. @@ -106,7 +112,7 @@ tcp-backlog 511 # incoming connections. There is no default, so Redis will not listen # on a unix socket when not specified. # -# unixsocket /tmp/redis.sock +# unixsocket /run/keydb.sock # unixsocketperm 700 # Close the connection after a client is idle for N seconds (0 to disable) @@ -118,8 +124,8 @@ timeout 0 # of communication. This is useful for two reasons: # # 1) Detect dead peers. -# 2) Take the connection alive from the point of view of network -# equipment in the middle. +# 2) Force network equipment in the middle to consider the connection to be +# alive. # # On Linux, the specified value (in seconds) is the period used to send ACKs. # Note that to close the connection the double of the time is needed. @@ -129,33 +135,148 @@ timeout 0 # Redis default starting with Redis 3.2.1. tcp-keepalive 300 +################################# TLS/SSL ##################################### + +# By default, TLS/SSL is disabled. To enable it, the "tls-port" configuration +# directive can be used to define TLS-listening ports. To enable TLS on the +# default port, use: +# +# port 0 +# tls-port 6379 + +# Configure a X.509 certificate and private key to use for authenticating the +# server to connected clients, masters or cluster peers. These files should be +# PEM formatted. +# +# tls-cert-file keydb.crt +# tls-key-file keydb.key + +# Normally Redis uses the same certificate for both server functions (accepting +# connections) and client functions (replicating from a master, establishing +# cluster bus connections, etc.). +# +# Sometimes certificates are issued with attributes that designate them as +# client-only or server-only certificates. In that case it may be desired to use +# different certificates for incoming (server) and outgoing (client) +# connections. To do that, use the following directives: +# +# tls-client-cert-file client.crt +# tls-client-key-file client.key + +# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: +# +# tls-dh-params-file keydb.dh + +# Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL +# clients and peers. Redis requires an explicit configuration of at least one +# of these, and will not implicitly use the system wide configuration. +# +# tls-ca-cert-file ca.crt +# tls-ca-cert-dir /etc/ssl/certs + +# By default, clients (including replica servers) on a TLS port are required +# to authenticate using valid client side certificates. +# +# If "no" is specified, client certificates are not required and not accepted. +# If "optional" is specified, client certificates are accepted and must be +# valid if provided, but are not required. +# +# tls-auth-clients no +# tls-auth-clients optional + +# By default, a Redis replica does not attempt to establish a TLS connection +# with its master. +# +# Use the following directive to enable TLS on replication links. +# +# tls-replication yes + +# By default, the Redis Cluster bus uses a plain TCP connection. To enable +# TLS for the bus protocol, use the following directive: +# +# tls-cluster yes + +# By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended +# that older formally deprecated versions are kept disabled to reduce the attack surface. +# You can explicitly specify TLS versions to support. +# Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", +# "TLSv1.3" (OpenSSL >= 1.1.1) or any combination. +# To enable only TLSv1.2 and TLSv1.3, use: +# +# tls-protocols "TLSv1.2 TLSv1.3" + +# Configure allowed ciphers. See the ciphers(1ssl) manpage for more information +# about the syntax of this string. +# +# Note: this configuration applies only to <= TLSv1.2. +# +# tls-ciphers DEFAULT:!MEDIUM + +# Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more +# information about the syntax of this string, and specifically for TLSv1.3 +# ciphersuites. +# +# tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 + +# When choosing a cipher, use the server's preference instead of the client +# preference. By default, the server follows the client's preference. +# +# tls-prefer-server-ciphers yes + +# By default, TLS session caching is enabled to allow faster and less expensive +# reconnections by clients that support it. Use the following directive to disable +# caching. +# +# tls-session-caching no + +# Change the default number of TLS sessions cached. A zero value sets the cache +# to unlimited size. The default size is 20480. +# +# tls-session-cache-size 5000 + +# Change the default timeout of cached TLS sessions. The default timeout is 300 +# seconds. +# +# tls-session-cache-timeout 60 + ################################# GENERAL ##################################### # By default Redis does not run as a daemon. Use 'yes' if you need it. -# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. -daemonize yes +# Note that Redis will write a pid file in /var/run/keydb.pid when daemonized. +# When Redis is supervised by upstart or systemd, this parameter has no impact. +daemonize no # If you run Redis from upstart or systemd, Redis can interact with your # supervision tree. Options: # supervised no - no supervision interaction # supervised upstart - signal upstart by putting Redis into SIGSTOP mode +# requires "expect stop" in your upstart job config # supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET +# on startup, and updating Redis status on a regular +# basis. # supervised auto - detect upstart or systemd method based on # UPSTART_JOB or NOTIFY_SOCKET environment variables # Note: these supervision methods only signal "process is ready." -# They do not enable continuous liveness pings back to your supervisor. -supervised no +# They do not enable continuous pings back to your supervisor. +# +# The default is "no". To run under upstart/systemd, you can simply uncomment +# the line below: +# +# supervised auto # If a pid file is specified, Redis writes it where specified at startup # and removes it at exit. # # When the server runs non daemonized, no pid file is created if none is # specified in the configuration. When the server is daemonized, the pid file -# is used even if not specified, defaulting to "/var/run/redis.pid". +# is used even if not specified, defaulting to "/var/run/keydb.pid". # # Creating a pid file is best effort: if Redis is not able to create it # nothing bad happens, the server will start and run normally. -pidfile /var/run/keydb/keydb-server.pid +# +# Note that on modern Linux systems "/run/keydb.pid" is more conforming +# and should be used instead. +pidfile /var/run/keydb_6379.pid # Specify the server verbosity level. # This can be one of: @@ -175,49 +296,81 @@ logfile /var/log/keydb/keydb-server.log # syslog-enabled no # Specify the syslog identity. -# syslog-ident redis +# syslog-ident keydb # Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. # syslog-facility local0 +# To disable the built in crash log, which will possibly produce cleaner core +# dumps when they are needed, uncomment the following: +# +# crash-log-enabled no + +# To disable the fast memory check that's run as part of the crash log, which +# will possibly let keydb terminate sooner, uncomment the following: +# +# crash-memcheck-enabled no + # Set the number of databases. The default database is DB 0, you can select # a different one on a per-connection basis using SELECT where # dbid is a number between 0 and 'databases'-1 databases 16 # By default Redis shows an ASCII art logo only when started to log to the -# standard output and if the standard output is a TTY. Basically this means -# that normally a logo is displayed only in interactive sessions. +# standard output and if the standard output is a TTY and syslog logging is +# disabled. Basically this means that normally a logo is displayed only in +# interactive sessions. # # However it is possible to force the pre-4.0 behavior and always show a # ASCII art logo in startup logs by setting the following option to yes. -always-show-logo yes +always-show-logo no + +# By default, Redis modifies the process title (as seen in 'top' and 'ps') to +# provide some runtime information. It is possible to disable this and leave +# the process name as executed by setting the following to no. +set-proc-title yes + +# When changing the process title, Redis uses the following template to construct +# the modified title. +# +# Template variables are specified in curly brackets. The following variables are +# supported: +# +# {title} Name of process as executed if parent, or type of child process. +# {listen-addr} Bind address or '*' followed by TCP or TLS port listening on, or +# Unix socket if only that's available. +# {server-mode} Special mode, i.e. "[sentinel]" or "[cluster]". +# {port} TCP port listening on, or 0. +# {tls-port} TLS port listening on, or 0. +# {unixsocket} Unix domain socket listening on, or "". +# {config-file} Name of configuration file used. +# +proc-title-template "{title} {listen-addr} {server-mode}" ################################ SNAPSHOTTING ################################ -# -# Save the DB on disk: -# -# save -# -# Will save the DB if both the given number of seconds and the given -# number of write operations against the DB occurred. -# -# In the example below the behaviour will be to save: -# after 900 sec (15 min) if at least 1 key changed -# after 300 sec (5 min) if at least 10 keys changed -# after 60 sec if at least 10000 keys changed -# -# Note: you can disable saving completely by commenting out all "save" lines. -# -# It is also possible to remove all the previously configured save -# points by adding a save directive with a single empty string argument -# like in the following example: -# -# save "" -save 900 1 -save 300 10 -save 60 10000 +# Save the DB to disk. +# +# save +# +# Redis will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. +# +# Snapshotting can be completely disabled with a single empty string argument +# as in following example: +# +# save "" +# +# Unless specified otherwise, by default Redis will save the DB: +# * After 3600 seconds (an hour) if at least 1 key changed +# * After 300 seconds (5 minutes) if at least 100 keys changed +# * After 60 seconds if at least 10000 keys changed +# +# You can set these explicitly by uncommenting the three following lines. +# +# save 3600 1 +# save 300 100 +# save 60 10000 # By default Redis will stop accepting writes if RDB snapshots are enabled # (at least one save point) and the latest background save failed. @@ -235,7 +388,7 @@ save 60 10000 stop-writes-on-bgsave-error yes # Compress string objects using LZF when dump .rdb databases? -# For default that's set to 'yes' as it's almost always a win. +# By default compression is enabled as it's almost always a win. # If you want to save some CPU in the saving child set it to 'no' but # the dataset will likely be bigger if you have compressible values or keys. rdbcompression yes @@ -249,9 +402,37 @@ rdbcompression yes # tell the loading code to skip the check. rdbchecksum yes +# Enables or disables full sanitation checks for ziplist and listpack etc when +# loading an RDB or RESTORE payload. This reduces the chances of a assertion or +# crash later on while processing commands. +# Options: +# no - Never perform full sanitation +# yes - Always perform full sanitation +# clients - Perform full sanitation only for user connections. +# Excludes: RDB files, RESTORE commands received from the master +# connection, and client connections which have the +# skip-sanitize-payload ACL flag. +# The default should be 'clients' but since it currently affects cluster +# resharding via MIGRATE, it is temporarily set to 'no' by default. +# +# sanitize-dump-payload no + # The filename where to dump the DB dbfilename dump.rdb +# Remove RDB files used by replication in instances without persistence +# enabled. By default this option is disabled, however there are environments +# where for regulations or other security concerns, RDB files persisted on +# disk by masters in order to feed replicas, or stored on disk by replicas +# in order to load them for the initial synchronization, should be deleted +# ASAP. Note that this option ONLY WORKS in instances that have both AOF +# and RDB persistence disabled, otherwise is completely ignored. +# +# An alternative (and sometimes better) way to obtain the same effect is +# to use diskless replication on both master and replicas instances. However +# in the case of replicas, diskless is not always an option. +rdb-del-sync-files no + # The working directory. # # The DB will be written inside this directory, with the filename specified @@ -310,11 +491,11 @@ dir /var/lib/keydb # still reply to client requests, possibly with out of date data, or the # data set may just be empty if this is the first synchronization. # -# 2) if replica-serve-stale-data is set to 'no' the replica will reply with -# an error "SYNC with master in progress" to all the kind of commands -# but to INFO, replicaOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, -# SUBSCRIBE, UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, -# COMMAND, POST, HOST: and LATENCY. +# 2) If replica-serve-stale-data is set to 'no' the replica will reply with +# an error "SYNC with master in progress" to all commands except: +# INFO, REPLICAOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, SUBSCRIBE, +# UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, COMMAND, POST, +# HOST and LATENCY. # replica-serve-stale-data yes @@ -377,9 +558,35 @@ repl-diskless-sync no # it entirely just set it to 0 seconds and the transfer will start ASAP. repl-diskless-sync-delay 5 -# Replicas send PINGs to server in a predefined interval. It's possible to change -# this interval with the repl_ping_replica_period option. The default value is 10 -# seconds. +# ----------------------------------------------------------------------------- +# WARNING: RDB diskless load is experimental. Since in this setup the replica +# does not immediately store an RDB on disk, it may cause data loss during +# failovers. RDB diskless load + Redis modules not handling I/O reads may also +# cause Redis to abort in case of I/O errors during the initial synchronization +# stage with the master. Use only if you know what you are doing. +# ----------------------------------------------------------------------------- +# +# Replica can load the RDB it reads from the replication link directly from the +# socket, or store the RDB to a file and read that file after it was completely +# received from the master. +# +# In many cases the disk is slower than the network, and storing and loading +# the RDB file may increase replication time (and even increase the master's +# Copy on Write memory and salve buffers). +# However, parsing the RDB file directly from the socket may mean that we have +# to flush the contents of the current database before the full rdb was +# received. For this reason we have the following options: +# +# "disabled" - Don't use diskless load (store the rdb file to the disk first) +# "on-empty-db" - Use diskless load only when it is completely safe. +# "swapdb" - Keep a copy of the current db contents in RAM while parsing +# the data directly from the socket. note that this requires +# sufficient memory, if you don't have it, you risk an OOM kill. +repl-diskless-load disabled + +# Replicas send PINGs to server in a predefined interval. It's possible to +# change this interval with the repl_ping_replica_period option. The default +# value is 10 seconds. # # repl-ping-replica-period 10 @@ -391,7 +598,8 @@ repl-diskless-sync-delay 5 # # It is important to make sure that this value is greater than the value # specified for repl-ping-replica-period otherwise a timeout will be detected -# every time there is low traffic between the master and the replica. +# every time there is low traffic between the master and the replica. The default +# value is 60 seconds. # # repl-timeout 60 @@ -416,21 +624,21 @@ repl-disable-tcp-nodelay no # resync is enough, just passing the portion of data the replica missed while # disconnected. # -# The bigger the replication backlog, the longer the time the replica can be -# disconnected and later be able to perform a partial resynchronization. +# The bigger the replication backlog, the longer the replica can endure the +# disconnect and later be able to perform a partial resynchronization. # -# The backlog is only allocated once there is at least a replica connected. +# The backlog is only allocated if there is at least one replica connected. # # repl-backlog-size 1mb -# After a master has no longer connected replicas for some time, the backlog -# will be freed. The following option configures the amount of seconds that -# need to elapse, starting from the time the last replica disconnected, for -# the backlog buffer to be freed. +# After a master has no connected replicas for some time, the backlog will be +# freed. The following option configures the amount of seconds that need to +# elapse, starting from the time the last replica disconnected, for the backlog +# buffer to be freed. # # Note that replicas never free the backlog for timeout, since they may be # promoted to masters later, and should be able to correctly "partially -# resynchronize" with the replicas: hence they should always accumulate backlog. +# resynchronize" with other replicas: hence they should always accumulate backlog. # # A value of 0 means to never release the backlog. # @@ -480,8 +688,8 @@ replica-priority 100 # Another place where this info is available is in the output of the # "ROLE" command of a master. # -# The listed IP and address normally reported by a replica is obtained -# in the following way: +# The listed IP address and port normally reported by a replica is +# obtained in the following way: # # IP: The address is auto detected by checking the peer address # of the socket used by the replica to connect with the master. @@ -491,7 +699,7 @@ replica-priority 100 # listen for connections. # # However when port forwarding or Network Address Translation (NAT) is -# used, the replica may be actually reachable via different IP and port +# used, the replica may actually be reachable via different IP and port # pairs. The following two options can be used by a replica in order to # report to its master a specific set of IP and port, so that both INFO # and ROLE will report those values. @@ -502,9 +710,45 @@ replica-priority 100 # replica-announce-ip 5.5.5.5 # replica-announce-port 1234 +############################### KEYS TRACKING ################################# + +# Redis implements server assisted support for client side caching of values. +# This is implemented using an invalidation table that remembers, using +# a radix key indexed by key name, what clients have which keys. In turn +# this is used in order to send invalidation messages to clients. Please +# check this page to understand more about the feature: +# +# https://redis.io/topics/client-side-caching +# +# When tracking is enabled for a client, all the read only queries are assumed +# to be cached: this will force Redis to store information in the invalidation +# table. When keys are modified, such information is flushed away, and +# invalidation messages are sent to the clients. However if the workload is +# heavily dominated by reads, Redis could use more and more memory in order +# to track the keys fetched by many clients. +# +# For this reason it is possible to configure a maximum fill value for the +# invalidation table. By default it is set to 1M of keys, and once this limit +# is reached, Redis will start to evict keys in the invalidation table +# even if they were not modified, just to reclaim memory: this will in turn +# force the clients to invalidate the cached values. Basically the table +# maximum size is a trade off between the memory you want to spend server +# side to track information about who cached what, and the ability of clients +# to retain cached objects in memory. +# +# If you set the value to 0, it means there are no limits, and Redis will +# retain as many keys as needed in the invalidation table. +# In the "stats" INFO section, you can find information about the number of +# keys in the invalidation table at every given moment. +# +# Note: when key tracking is used in broadcasting mode, no memory is used +# in the server side so this setting is useless. +# +# tracking-table-max-keys 1000000 + ################################## SECURITY ################################### -# Warning: since Redis is pretty fast an outside user can try up to +# Warning: since Redis is pretty fast, an outside user can try up to # 1 million passwords per second against a modern box. This means that you # should use very strong passwords, otherwise they will be very easy to break. # Note that because the password is really a shared secret between the client @@ -528,12 +772,14 @@ replica-priority 100 # AUTH (or the HELLO command AUTH option) in order to be authenticated and # start to work. # -# The ACL rules that describe what an user can do are the following: +# The ACL rules that describe what a user can do are the following: # # on Enable the user: it is possible to authenticate as this user. # off Disable the user: it's no longer possible to authenticate # with this user, however the already authenticated connections # will still work. +# skip-sanitize-payload RESTORE dump-payload sanitation is skipped. +# sanitize-payload RESTORE dump-payload is sanitized (default). # + Allow the execution of that command # - Disallow the execution of that command # +@ Allow the execution of all the commands in such category @@ -556,7 +802,12 @@ replica-priority 100 # It is possible to specify multiple patterns. # allkeys Alias for ~* # resetkeys Flush the list of allowed keys patterns. -# > Add this passowrd to the list of valid password for the user. +# & Add a glob-style pattern of Pub/Sub channels that can be +# accessed by the user. It is possible to specify multiple channel +# patterns. +# allchannels Alias for &* +# resetchannels Flush the list of allowed channel patterns. +# > Add this password to the list of valid password for the user. # For example >mypass will add "mypass" to the list. # This directive clears the "nopass" flag (see later). # < Remove this password from the list of valid passwords. @@ -598,26 +849,58 @@ replica-priority 100 # For more information about ACL configuration please refer to # the Redis web site at https://redis.io/topics/acl +# ACL LOG +# +# The ACL Log tracks failed commands and authentication events associated +# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked +# by ACLs. The ACL Log is stored in memory. You can reclaim memory with +# ACL LOG RESET. Define the maximum entry length of the ACL Log below. +acllog-max-len 128 + # Using an external ACL file # # Instead of configuring users here in this file, it is possible to use # a stand-alone file just listing users. The two methods cannot be mixed: -# if you configure users here and at the same time you activate the exteranl +# if you configure users here and at the same time you activate the external # ACL file, the server will refuse to start. # # The format of the external ACL user file is exactly the same as the -# format that is used inside redis.conf to describe users. +# format that is used inside keydb.conf to describe users. # -# aclfile /etc/redis/users.acl +# aclfile /etc/keydb/users.acl -# IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatiblity +# IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatibility # layer on top of the new ACL system. The option effect will be just setting # the password for the default user. Clients will still authenticate using # AUTH as usually, or more explicitly with AUTH default # if they follow the new protocol: both will work. # +# The requirepass is not compatable with aclfile option and the ACL LOAD +# command, these will cause requirepass to be ignored. +# # requirepass foobared +# New users are initialized with restrictive permissions by default, via the +# equivalent of this ACL rule 'off resetkeys -@all'. Starting with Redis 6.2, it +# is possible to manage access to Pub/Sub channels with ACL rules as well. The +# default Pub/Sub channels permission if new users is controlled by the +# acl-pubsub-default configuration directive, which accepts one of these values: +# +# allchannels: grants access to all Pub/Sub channels +# resetchannels: revokes access to all Pub/Sub channels +# +# To ensure backward compatibility while upgrading Redis 6.0, acl-pubsub-default +# defaults to the 'allchannels' permission. +# +# Future compatibility note: it is very likely that in a future version of Redis +# the directive's default of 'allchannels' will be changed to 'resetchannels' in +# order to provide better out-of-the-box Pub/Sub security. Therefore, it is +# recommended that you explicitly define Pub/Sub permissions for all users +# rather then rely on implicit default values. Once you've set explicit +# Pub/Sub for all exisitn users, you should uncomment the following line. +# +# acl-pubsub-default resetchannels + # Command renaming (DEPRECATED). # # ------------------------------------------------------------------------ @@ -654,6 +937,11 @@ replica-priority 100 # Once the limit is reached Redis will close all the new connections sending # an error 'max number of clients reached'. # +# IMPORTANT: When Redis Cluster is used, the max number of connections is also +# shared with the cluster bus: every node in the cluster will use two +# connections, one incoming and another outgoing. It is important to size the +# limit accordingly in case of very large clusters. +# # maxclients 10000 ############################## MEMORY MANAGEMENT ################################ @@ -701,14 +989,12 @@ replica-priority 100 # Both LRU, LFU and volatile-ttl are implemented using approximated # randomized algorithms. # -# Note: with any of the above policies, Redis will return an error on write -# operations, when there are no suitable keys for eviction. -# -# At the date of writing these commands are: set setnx setex append -# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd -# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby -# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby -# getset mset msetnx exec sort +# Note: with any of the above policies, when there are no suitable keys for +# eviction, Redis will return an error on write operations that require +# more memory. These are usually commands that create new keys, add data or +# modify existing keys. A few examples are: SET, INCR, HSET, LPUSH, SUNIONSTORE, +# SORT (due to the STORE argument), and EXEC (if the transaction includes any +# command that requires memory). # # The default is: # @@ -716,8 +1002,8 @@ replica-priority 100 # LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated # algorithms (in order to save memory), so you can tune it for speed or -# accuracy. For default Redis will check five keys and pick the one that was -# used less recently, you can change the sample size using the following +# accuracy. By default Redis will check five keys and pick the one that was +# used least recently, you can change the sample size using the following # configuration directive. # # The default of 5 produces good enough results. 10 Approximates very closely @@ -725,6 +1011,14 @@ replica-priority 100 # # maxmemory-samples 5 +# Eviction processing is designed to function well with the default setting. +# If there is an unusually large amount of write traffic, this value may need to +# be increased. Decreasing this value may reduce latency at the risk of +# eviction processing effectiveness +# 0 = minimum latency, 10 = default, 100 = process without regard to latency +# +# maxmemory-eviction-tenacity 10 + # Starting from Redis 5, by default a replica will ignore its maxmemory setting # (unless it is promoted to master after a failover or manually). It means # that the eviction of keys will be just handled by the master, sending the @@ -745,6 +1039,23 @@ replica-priority 100 # # replica-ignore-maxmemory yes +# Redis reclaims expired keys in two ways: upon access when those keys are +# found to be expired, and also in background, in what is called the +# "active expire key". The key space is slowly and interactively scanned +# looking for expired keys to reclaim, so that it is possible to free memory +# of keys that are expired and will never be accessed again in a short time. +# +# The default effort of the expire cycle will try to avoid having more than +# ten percent of expired keys still in memory, and will try to avoid consuming +# more than 25% of total memory and to add latency to the system. However +# it is possible to increase the expire "effort" that is normally set to +# "1", to a greater value, up to the value "10". At its maximum value the +# system will use more CPU, longer cycles (and technically may introduce +# more latency), and will tolerate less already expired keys still present +# in the system. It's a tradeoff between memory, CPU and latency. +# +# active-expire-effort 1 + ############################# LAZY FREEING #################################### # Redis has two primitives to delete keys. One is called DEL and is a blocking @@ -787,13 +1098,117 @@ replica-priority 100 # In all the above cases the default is to delete objects in a blocking way, # like if DEL was called. However you can configure each case specifically # in order to instead release memory in a non-blocking way like if UNLINK -# was called, using the following configuration directives: +# was called, using the following configuration directives. lazyfree-lazy-eviction no lazyfree-lazy-expire no lazyfree-lazy-server-del no replica-lazy-flush no +# It is also possible, for the case when to replace the user code DEL calls +# with UNLINK calls is not easy, to modify the default behavior of the DEL +# command to act exactly like UNLINK, using the following configuration +# directive: + +lazyfree-lazy-user-del no + +# FLUSHDB, FLUSHALL, and SCRIPT FLUSH support both asynchronous and synchronous +# deletion, which can be controlled by passing the [SYNC|ASYNC] flags into the +# commands. When neither flag is passed, this directive will be used to determine +# if the data should be deleted asynchronously. + +lazyfree-lazy-user-flush no + +################################ THREADED I/O ################################# + +# Redis is mostly single threaded, however there are certain threaded +# operations such as UNLINK, slow I/O accesses and other things that are +# performed on side threads. +# +# Now it is also possible to handle Redis clients socket reads and writes +# in different I/O threads. Since especially writing is so slow, normally +# Redis users use pipelining in order to speed up the Redis performances per +# core, and spawn multiple instances in order to scale more. Using I/O +# threads it is possible to easily speedup two times Redis without resorting +# to pipelining nor sharding of the instance. +# +# By default threading is disabled, we suggest enabling it only in machines +# that have at least 4 or more cores, leaving at least one spare core. +# Using more than 8 threads is unlikely to help much. We also recommend using +# threaded I/O only if you actually have performance problems, with Redis +# instances being able to use a quite big percentage of CPU time, otherwise +# there is no point in using this feature. +# +# So for instance if you have a four cores boxes, try to use 2 or 3 I/O +# threads, if you have a 8 cores, try to use 6 threads. In order to +# enable I/O threads use the following configuration directive: +# +# io-threads 4 +# +# Setting io-threads to 1 will just use the main thread as usual. +# When I/O threads are enabled, we only use threads for writes, that is +# to thread the write(2) syscall and transfer the client buffers to the +# socket. However it is also possible to enable threading of reads and +# protocol parsing using the following configuration directive, by setting +# it to yes: +# +# io-threads-do-reads no +# +# Usually threading reads doesn't help much. +# +# NOTE 1: This configuration directive cannot be changed at runtime via +# CONFIG SET. Aso this feature currently does not work when SSL is +# enabled. +# +# NOTE 2: If you want to test the Redis speedup using keydb-benchmark, make +# sure you also run the benchmark itself in threaded mode, using the +# --threads option to match the number of Redis threads, otherwise you'll not +# be able to notice the improvements. + +############################ KERNEL OOM CONTROL ############################## + +# On Linux, it is possible to hint the kernel OOM killer on what processes +# should be killed first when out of memory. +# +# Enabling this feature makes Redis actively control the oom_score_adj value +# for all its processes, depending on their role. The default scores will +# attempt to have background child processes killed before all others, and +# replicas killed before masters. +# +# Redis supports three options: +# +# no: Don't make changes to oom-score-adj (default). +# yes: Alias to "relative" see below. +# absolute: Values in oom-score-adj-values are written as is to the kernel. +# relative: Values are used relative to the initial value of oom_score_adj when +# the server starts and are then clamped to a range of -1000 to 1000. +# Because typically the initial value is 0, they will often match the +# absolute values. +oom-score-adj no + +# When oom-score-adj is used, this directive controls the specific values used +# for master, replica and background child processes. Values range -2000 to +# 2000 (higher means more likely to be killed). +# +# Unprivileged processes (not root, and without CAP_SYS_RESOURCE capabilities) +# can freely increase their value, but not decrease it below its initial +# settings. This means that setting oom-score-adj to "relative" and setting the +# oom-score-adj-values to positive values will always succeed. +oom-score-adj-values 0 200 800 + + +#################### KERNEL transparent hugepage CONTROL ###################### + +# Usually the kernel Transparent Huge Pages control is set to "madvise" or +# or "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which +# case this config has no effect. On systems in which it is set to "always", +# keydb will attempt to disable it specifically for the keydb process in order +# to avoid latency problems specifically with fork(2) and CoW. +# If for some reason you prefer to keep it enabled, you can set this config to +# "no" and the kernel global to "always". + +disable-thp yes + ############################## APPEND ONLY MODE ############################### # By default Redis asynchronously dumps the dataset on disk. This mode is @@ -918,8 +1333,8 @@ aof-load-truncated yes # # [RDB file][AOF tail] # -# When loading Redis recognizes that the AOF file starts with the "REDIS" -# string and loads the prefixed RDB file, and continues loading the AOF +# When loading, Redis recognizes that the AOF file starts with the "REDIS" +# string and loads the prefixed RDB file, then continues loading the AOF # tail. aof-use-rdb-preamble yes @@ -933,7 +1348,7 @@ aof-use-rdb-preamble yes # # When a long running script exceeds the maximum execution time only the # SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be -# used to stop a script that did not yet called write commands. The second +# used to stop a script that did not yet call any write commands. The second # is the only way to shut down the server in the case a write command was # already issued by the script but the user doesn't want to wait for the natural # termination of the script. @@ -965,7 +1380,7 @@ lua-time-limit 5000 # Cluster node timeout is the amount of milliseconds a node must be unreachable # for it to be considered in failure state. -# Most other internal time limits are multiple of the node timeout. +# Most other internal time limits are a multiple of the node timeout. # # cluster-node-timeout 15000 @@ -992,18 +1407,18 @@ lua-time-limit 5000 # the failover if, since the last interaction with the master, the time # elapsed is greater than: # -# (node-timeout * replica-validity-factor) + repl-ping-replica-period +# (node-timeout * cluster-replica-validity-factor) + repl-ping-replica-period # -# So for example if node-timeout is 30 seconds, and the replica-validity-factor +# So for example if node-timeout is 30 seconds, and the cluster-replica-validity-factor # is 10, and assuming a default repl-ping-replica-period of 10 seconds, the # replica will not try to failover if it was not able to talk with the master # for longer than 310 seconds. # -# A large replica-validity-factor may allow replicas with too old data to failover +# A large cluster-replica-validity-factor may allow replicas with too old data to failover # a master, while a too small value may prevent the cluster from being able to # elect a replica at all. # -# For maximum availability, it is possible to set the replica-validity-factor +# For maximum availability, it is possible to set the cluster-replica-validity-factor # to a value of 0, which means, that replicas will always try to failover the # master regardless of the last time they interacted with the master. # (However they'll always try to apply a delay proportional to their @@ -1034,7 +1449,7 @@ lua-time-limit 5000 # cluster-migration-barrier 1 # By default Redis Cluster nodes stop accepting queries if they detect there -# is at least an hash slot uncovered (no available node is serving it). +# is at least a hash slot uncovered (no available node is serving it). # This way if the cluster is partially down (for example a range of hash slots # are no longer covered) all the cluster becomes, eventually, unavailable. # It automatically returns available as soon as all the slots are covered again. @@ -1047,7 +1462,7 @@ lua-time-limit 5000 # cluster-require-full-coverage yes # This option, when set to yes, prevents replicas from trying to failover its -# master during master failures. However the master can still perform a +# master during master failures. However the replica can still perform a # manual failover, if forced to do so. # # This is useful in different scenarios, especially in the case of multiple @@ -1073,7 +1488,7 @@ lua-time-limit 5000 # * cluster-announce-port # * cluster-announce-bus-port # -# Each instruct the node about its address, client port, and cluster message +# Each instructs the node about its address, client port, and cluster message # bus port. The information is then published in the header of the bus packets # so that other nodes will be able to correctly map the address of the node # publishing the information. @@ -1084,7 +1499,7 @@ lua-time-limit 5000 # Note that when remapped, the bus port may not be at the fixed offset of # clients port + 10000, so you can specify any port and bus-port depending # on how they get remapped. If the bus-port is not set, a fixed offset of -# 10000 will be used as usually. +# 10000 will be used as usual. # # Example: # @@ -1162,7 +1577,11 @@ latency-monitor-threshold 0 # z Sorted set commands # x Expired events (events generated every time a key expires) # e Evicted events (events generated when a key is evicted for maxmemory) -# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# t Stream commands +# m Key-miss events (Note: It is not included in the 'A' class) +# A Alias for g$lshzxet, so that the "AKE" string means all the events +# (Except key-miss events which are excluded from 'A' due to their +# unique nature). # # The "notify-keyspace-events" takes as argument a string that is composed # of zero or multiple characters. The empty string means that notifications @@ -1209,7 +1628,7 @@ notify-keyspace-events "" # two kind of inline requests that were anyway illegal: an empty request # or any request that starts with "/" (there are no Redis commands starting # with such a slash). Normal RESP2/RESP3 requests are completely out of the -# path of the Gopher protocol implementation and are served as usually as well. +# path of the Gopher protocol implementation and are served as usual as well. # # If you open a connection to Redis when Gopher is enabled and send it # a string like "/foo", if there is a key named "/foo" it is served via the @@ -1233,8 +1652,11 @@ notify-keyspace-events "" # # So use the 'requirepass' option to protect your instance. # -# To enable Gopher support uncomment the following line and set -# the option from no (the default) to yes. +# Note that Gopher is not currently supported when 'io-threads-do-reads' +# is enabled. +# +# To enable Gopher support, uncomment the following line and set the option +# from no (the default) to yes. # # gopher-enabled no @@ -1310,7 +1732,7 @@ hll-sparse-max-bytes 3000 # maximum number of items it may contain before switching to a new node when # appending new stream entries. If any of the following settings are set to # zero, the limit is ignored, so for instance it is possible to set just a -# max entires limit by setting max-bytes to 0 and max-entries to the desired +# max entries limit by setting max-bytes to 0 and max-entries to the desired # value. stream-node-max-bytes 4096 stream-node-max-entries 100 @@ -1381,8 +1803,8 @@ client-output-buffer-limit pubsub 32mb 8mb 60 # client-query-buffer-limit 1gb # In the Redis protocol, bulk requests, that are, elements representing single -# strings, are normally limited ot 512 mb. However you can change this limit -# here. +# strings, are normally limited to 512 mb. However you can change this limit +# here, but must be 1mb or greater # # proto-max-bulk-len 512mb @@ -1410,9 +1832,9 @@ hz 10 # # Since the default HZ value by default is conservatively set to 10, Redis # offers, and enables by default, the ability to use an adaptive HZ value -# which will temporary raise when there are many connected clients. +# which will temporarily raise when there are many connected clients. # -# When dynamic HZ is enabled, the actual configured HZ will be used as +# When dynamic HZ is enabled, the actual configured HZ will be used # as a baseline, but multiples of the configured HZ value will be actually # used as needed once more clients are connected. In this way an idle # instance will use very little CPU time while a busy instance will be @@ -1425,7 +1847,7 @@ dynamic-hz yes # big latency spikes. aof-rewrite-incremental-fsync yes -# When redis saves RDB file, if the following option is enabled +# When KeyDB saves RDB file, if the following option is enabled # the file will be fsync-ed every 32 MB of data generated. This is useful # in order to commit the file to the disk more incrementally and avoid # big latency spikes. @@ -1477,7 +1899,7 @@ rdb-save-incremental-fsync yes # for the key counter to be divided by two (or decremented if it has a value # less <= 10). # -# The default value for the lfu-decay-time is 1. A Special value of 0 means to +# The default value for the lfu-decay-time is 1. A special value of 0 means to # decay the counter every time it happens to be scanned. # # lfu-log-factor 10 @@ -1501,7 +1923,7 @@ rdb-save-incremental-fsync yes # restart is needed in order to lower the fragmentation, or at least to flush # away all the data and create it again. However thanks to this feature # implemented by Oran Agra for Redis 4.0 this process can happen at runtime -# in an "hot" way, while the server is running. +# in a "hot" way, while the server is running. # # Basically when the fragmentation is over a certain level (see the # configuration options below) Redis will start to create new copies of the @@ -1549,6 +1971,42 @@ rdb-save-incremental-fsync yes # the main dictionary scan # active-defrag-max-scan-fields 1000 +# Jemalloc background thread for purging will be enabled by default +jemalloc-bg-thread yes + +# It is possible to pin different threads and processes of Redis to specific +# CPUs in your system, in order to maximize the performances of the server. +# This is useful both in order to pin different Redis threads in different +# CPUs, but also in order to make sure that multiple Redis instances running +# in the same host will be pinned to different CPUs. +# +# Normally you can do this using the "taskset" command, however it is also +# possible to this via Redis configuration directly, both in Linux and FreeBSD. +# +# You can pin the server/IO threads, bio threads, aof rewrite child process, and +# the bgsave child process. The syntax to specify the cpu list is the same as +# the taskset command: +# +# Set KeyDB server/io threads to cpu affinity 0,2,4,6: +# server_cpulist 0-7:2 +# +# Set bio threads to cpu affinity 1,3: +# bio_cpulist 1,3 +# +# Set aof rewrite child process to cpu affinity 8,9,10,11: +# aof_rewrite_cpulist 8-11 +# +# Set bgsave child process to cpu affinity 1,10,11 +# bgsave_cpulist 1,10-11 + +# In some cases KeyDB will emit warnings and even refuse to start if it detects +# that the system is in bad state, it is possible to suppress these warnings +# by setting the following config which takes a space delimited list of warnings +# to suppress +# +# ignore-warnings ARM64-COW-BUG + + # Path to directory for file backed scratchpad. The file backed scratchpad # reduces memory requirements by storing rarely accessed data on disk # instead of RAM. A temporary file will be created in this directory. @@ -1574,3 +2032,4 @@ server-threads 2 # Enable FLASH support? (Pro Only) # storage-provider flash /path/to/flash/db + diff --git a/runtest b/runtest index ade1bd09a..c6349d118 100755 --- a/runtest +++ b/runtest @@ -2,6 +2,8 @@ TCL_VERSIONS="8.5 8.6" TCLSH="" +export ASAN_OPTIONS=allocator_may_return_null=1 $ASAN_OPTIONS + for VERSION in $TCL_VERSIONS; do TCL=`which tclsh$VERSION 2>/dev/null` && TCLSH=$TCL done diff --git a/runtest-cluster b/runtest-cluster index 27829a5fe..a86e93141 100755 --- a/runtest-cluster +++ b/runtest-cluster @@ -8,7 +8,7 @@ done if [ -z $TCLSH ] then - echo "You need tcl 8.5 or newer in order to run the Redis Sentinel test" + echo "You need tcl 8.5 or newer in order to run the Redis Cluster test" exit 1 fi $TCLSH tests/cluster/run.tcl $* diff --git a/runtest-moduleapi b/runtest-moduleapi index 151e38c4f..2e5bffcba 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -1,6 +1,7 @@ #!/bin/sh TCL_VERSIONS="8.5 8.6" TCLSH="" +[ -z "$MAKE" ] && MAKE=make for VERSION in $TCL_VERSIONS; do TCL=`which tclsh$VERSION 2>/dev/null` && TCLSH=$TCL @@ -8,11 +9,11 @@ done if [ -z $TCLSH ] then - echo "You need tcl 8.5 or newer in order to run the Redis test" + echo "You need tcl 8.5 or newer in order to run the Redis ModuleApi test" exit 1 fi -make -C tests/modules && \ +$MAKE -C tests/modules && \ $TCLSH tests/test_helper.tcl \ --single unit/moduleapi/commandfilter \ --single unit/moduleapi/fork \ @@ -22,6 +23,7 @@ $TCLSH tests/test_helper.tcl \ --single unit/moduleapi/hooks \ --single unit/moduleapi/misc \ --single unit/moduleapi/blockonkeys \ +--single unit/moduleapi/blockonbackground \ --single unit/moduleapi/scan \ --single unit/moduleapi/datatype \ --single unit/moduleapi/auth \ @@ -30,5 +32,10 @@ $TCLSH tests/test_helper.tcl \ --single unit/moduleapi/moduleloadsave \ --single unit/moduleapi/getkeys \ --single unit/moduleapi/timers \ +--single unit/moduleapi/test_lazyfree \ +--single unit/moduleapi/defrag \ +--single unit/moduleapi/hash \ +--single unit/moduleapi/zset \ +--single unit/moduleapi/stream \ --config server-threads 3 \ "${@}" diff --git a/sentinel.conf b/sentinel.conf index 2ec6717d5..5ab5a8b5c 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -124,6 +124,42 @@ sentinel monitor mymaster 127.0.0.1 6379 2 # Default is 30 seconds. sentinel down-after-milliseconds mymaster 30000 +# IMPORTANT NOTE: starting with KeyDB 6.2 ACL capability is supported for +# Sentinel mode, please refer to the KeyDB website https://redis.io/topics/acl +# for more details. + +# Sentinel's ACL users are defined in the following format: +# +# user ... acl rules ... +# +# For example: +# +# user worker +@admin +@connection ~* on >ffa9203c493aa99 +# +# For more information about ACL configuration please refer to the Redis +# website at https://redis.io/topics/acl and redis server configuration +# template redis.conf. + +# ACL LOG +# +# The ACL Log tracks failed commands and authentication events associated +# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked +# by ACLs. The ACL Log is stored in memory. You can reclaim memory with +# ACL LOG RESET. Define the maximum entry length of the ACL Log below. +acllog-max-len 128 + +# Using an external ACL file +# +# Instead of configuring users here in this file, it is possible to use +# a stand-alone file just listing users. The two methods cannot be mixed: +# if you configure users here and at the same time you activate the external +# ACL file, the server will refuse to start. +# +# The format of the external ACL user file is exactly the same as the +# format that is used inside redis.conf to describe users. +# +# aclfile /etc/redis/sentinel-users.acl + # requirepass # # You can configure Sentinel itself to require a password, however when doing @@ -131,6 +167,29 @@ sentinel down-after-milliseconds mymaster 30000 # other Sentinels. So you need to configure all your Sentinels in a given # group with the same "requirepass" password. Check the following documentation # for more info: https://redis.io/topics/sentinel +# +# IMPORTANT NOTE: starting with Redis 6.2 "requirepass" is a compatibility +# layer on top of the ACL system. The option effect will be just setting +# the password for the default user. Clients will still authenticate using +# AUTH as usually, or more explicitly with AUTH default +# if they follow the new protocol: both will work. +# +# New config files are advised to use separate authentication control for +# incoming connections (via ACL), and for outgoing connections (via +# sentinel-user and sentinel-pass) +# +# The requirepass is not compatable with aclfile option and the ACL LOAD +# command, these will cause requirepass to be ignored. + +# sentinel sentinel-user +# +# You can configure Sentinel to authenticate with other Sentinels with specific +# user name. + +# sentinel sentinel-pass +# +# The password for Sentinel to authenticate with other Sentinels. If sentinel-user +# is not configured, Sentinel will use 'default' user with sentinel-pass to authenticate. # sentinel parallel-syncs # @@ -262,3 +321,21 @@ sentinel deny-scripts-reconfig yes # is possible to just rename a command to itself: # # SENTINEL rename-command mymaster CONFIG CONFIG + +# HOSTNAMES SUPPORT +# +# Normally Sentinel uses only IP addresses and requires SENTINEL MONITOR +# to specify an IP address. Also, it requires the Redis replica-announce-ip +# keyword to specify only IP addresses. +# +# You may enable hostnames support by enabling resolve-hostnames. Note +# that you must make sure your DNS is configured properly and that DNS +# resolution does not introduce very long delays. +# +SENTINEL resolve-hostnames no + +# When resolve-hostnames is enabled, Sentinel still uses IP addresses +# when exposing instances to users, configuration files, etc. If you want +# to retain the hostnames when announced, enable announce-hostnames below. +# +SENTINEL announce-hostnames no diff --git a/src/Makefile b/src/Makefile index 9e4070d4a..43c861acd 100644 --- a/src/Makefile +++ b/src/Makefile @@ -16,13 +16,15 @@ release_hdr := $(shell sh -c './mkreleasehdr.sh') uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not') OPTIMIZATION?=-O2 -DEPENDENCY_TARGETS=hiredis linenoise lua +DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram NODEPS:=clean distclean # Default settings -STD=-std=c11 -pedantic -DREDIS_STATIC='' +STD=-pedantic -DREDIS_STATIC='' CXX_STD=-std=c++14 -pedantic -fno-rtti -D__STDC_FORMAT_MACROS ifneq (,$(findstring clang,$(CC))) + STD+=-Wno-c11-extensions +else ifneq (,$(findstring FreeBSD,$(uname_S))) STD+=-Wno-c11-extensions endif @@ -30,6 +32,16 @@ endif WARN=-Wall -W -Wno-missing-field-initializers OPT=$(OPTIMIZATION) +# Detect if the compiler supports C11 _Atomic +C11_ATOMIC := $(shell sh -c 'echo "\#include " > foo.c; \ + $(CC) -std=c11 -c foo.c -o foo.o > /dev/null 2>&1; \ + if [ -f foo.o ]; then echo "yes"; rm foo.o; fi; rm foo.c') +ifeq ($(C11_ATOMIC),yes) + STD+=-std=c11 +else + STD+=-std=c99 +endif + PREFIX?=/usr/local INSTALL_BIN=$(PREFIX)/bin INSTALL=install @@ -117,10 +129,12 @@ endif ifeq ($(uname_S),SunOS) # SunOS - ifneq ($(@@),32bit) - CFLAGS+= -m64 + ifeq ($(findstring -m32,$(FINAL_CFLAGS)),) + CFLAGS+=-m64 CXXFLAGS+= -m64 - LDFLAGS+= -m64 + endif + ifeq ($(findstring -m32,$(FINAL_LDFLAGS)),) + LDFLAGS+=-m64 endif DEBUG=-g DEBUG_FLAGS=-g @@ -133,8 +147,18 @@ else ifeq ($(uname_S),Darwin) # Darwin FINAL_LIBS+= -ldl + # Homebrew's OpenSSL is not linked to /usr/local to avoid + # conflicts with the system's LibreSSL installation so it + # must be referenced explicitly during build. +ifeq ($(uname_M),arm64) + # Homebrew arm64 uses /opt/homebrew as HOMEBREW_PREFIX + OPENSSL_CFLAGS=-I/opt/homebrew/opt/openssl/include + OPENSSL_LDFLAGS=-L/opt/homebrew/opt/openssl/lib +else + # Homebrew x86/ppc uses /usr/local as HOMEBREW_PREFIX OPENSSL_CFLAGS=-I/usr/local/opt/openssl/include OPENSSL_LDFLAGS=-L/usr/local/opt/openssl/lib +endif else ifeq ($(uname_S),AIX) # AIX @@ -202,11 +226,13 @@ endif endif endif # Include paths to dependencies -FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -FINAL_CXXFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src +FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram +FINAL_CXXFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram # Determine systemd support and/or build preference (defaulting to auto-detection) BUILD_WITH_SYSTEMD=no +LIBSYSTEMD_LIBS=-lsystemd + # If 'USE_SYSTEMD' in the environment is neither "no" nor "yes", try to # auto-detect libsystemd's presence and link accordingly. ifneq ($(USE_SYSTEMD),no) @@ -215,17 +241,18 @@ ifneq ($(USE_SYSTEMD),no) # (unless a later check tells us otherwise) ifeq ($(LIBSYSTEMD_PKGCONFIG),0) BUILD_WITH_SYSTEMD=yes + LIBSYSTEMD_LIBS=$(shell $(PKG_CONFIG) --libs libsystemd) endif endif + +# If 'USE_SYSTEMD' is set to "yes" use pkg-config if available or fall back to +# default -lsystemd. ifeq ($(USE_SYSTEMD),yes) -ifneq ($(LIBSYSTEMD_PKGCONFIG),0) -$(error USE_SYSTEMD is set to "$(USE_SYSTEMD)", but $(PKG_CONFIG) cannot find libsystemd) -endif -# Force building with libsystemd BUILD_WITH_SYSTEMD=yes endif + ifeq ($(BUILD_WITH_SYSTEMD),yes) - FINAL_LIBS+=$(shell $(PKG_CONFIG) --libs libsystemd) + FINAL_LIBS+=$(LIBSYSTEMD_LIBS) FINAL_CFLAGS+= -DHAVE_LIBSYSTEMD endif @@ -256,10 +283,9 @@ ifeq ($(MALLOC),memkind) endif ifeq ($(BUILD_TLS),yes) - FINAL_CFLAGS+=-DUSE_OPENSSL $(OPENSSL_CXXFLAGS) - FINAL_CXXFLAGS+=-DUSE_OPENSSL $(OPENSSL_CXXFLAGS) - FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS) - FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a -lssl -lcrypto + FINAL_CFLAGS+=-DUSE_OPENSSL $(OPENSSL_CFLAGS) + FINAL_CXXFLAGS+=-DUSE_OPENSSL $(OPENSSL_CXXFLAGS) + FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS) LIBSSL_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libssl && echo $$?) ifeq ($(LIBSSL_PKGCONFIG),0) LIBSSL_LIBS=$(shell $(PKG_CONFIG) --libs libssl) @@ -296,11 +322,11 @@ endif REDIS_SERVER_NAME=keydb-server$(PROG_SUFFIX) REDIS_SENTINEL_NAME=keydb-sentinel$(PROG_SUFFIX) -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o t_nhash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o connection.o tls.o sha256.o motd.o timeout.o setcpuaffinity.o $(ASM_OBJ) +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o t_nhash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o connection.o tls.o sha256.o motd.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o $(ASM_OBJ) REDIS_CLI_NAME=keydb-cli$(PROG_SUFFIX) -REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o redis-cli-cpphelper.o zmalloc.o release.o anet.o ae.o crcspeed.o crc64.o siphash.o crc16.o storage-lite.o fastlock.o new.o motd.o $(ASM_OBJ) +REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o redis-cli-cpphelper.o zmalloc.o release.o anet.o ae.o crcspeed.o crc64.o siphash.o crc16.o storage-lite.o fastlock.o new.o motd.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ) REDIS_BENCHMARK_NAME=keydb-benchmark$(PROG_SUFFIX) -REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o redis-benchmark.o storage-lite.o fastlock.o new.o $(ASM_OBJ) +REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o release.o crcspeed.o crc64.o siphash.o redis-benchmark.o storage-lite.o fastlock.o new.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ) REDIS_CHECK_RDB_NAME=keydb-check-rdb$(PROG_SUFFIX) REDIS_CHECK_AOF_NAME=keydb-check-aof$(PROG_SUFFIX) @@ -374,9 +400,9 @@ $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ) # keydb-benchmark $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) - $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a $(FINAL_LIBS) + $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/hdr_histogram/hdr_histogram.o $(FINAL_LIBS) -dict-benchmark: dict.cpp zmalloc.cpp sds.c siphash.c +dict-benchmark: dict.cpp zmalloc.cpp sds.c siphash.c mt19937-64.c $(REDIS_CC) $(FINAL_CFLAGS) $^ -D DICT_BENCHMARK_MAIN -o $@ $(FINAL_LIBS) DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d) @@ -407,10 +433,10 @@ distclean: clean .PHONY: distclean -test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) +test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) @(cd ..; ./runtest) -test-sentinel: $(REDIS_SENTINEL_NAME) +test-sentinel: $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) @(cd ..; ./runtest-sentinel) check: test @@ -422,10 +448,6 @@ lcov: @genhtml --legend -o lcov-html KeyDB.info @genhtml --legend -o lcov-html KeyDB.info | grep lines | awk '{print $$2;}' | sed 's/%//g' -test-sds: sds.c sds.h - $(REDIS_CC) sds.c zmalloc.cpp -DSDS_TEST_MAIN $(FINAL_LIBS) -o /tmp/sds_test - /tmp/sds_test - .PHONY: lcov bench: $(REDIS_BENCHMARK_NAME) @@ -447,7 +469,7 @@ valgrind: $(MAKE) OPTIMIZATION="-O0" USEASM="false" MALLOC="libc" CFLAGS="-DSANITIZE" CXXFLAGS="-DSANITIZE" helgrind: - $(MAKE) OPTIMIZATION="-O0" MALLOC="libc" CFLAGS="-D__ATOMIC_VAR_FORCE_SYNC_MACROS" + $(MAKE) OPTIMIZATION="-O0" MALLOC="libc" CFLAGS="-D__ATOMIC_VAR_FORCE_SYNC_MACROS" REDIS_CFLAGS="-I/usr/local/include" REDIS_LDFLAGS="-L/usr/local/lib" src/help.h: @../utils/generate-command-help.rb > help.h @@ -457,8 +479,8 @@ install: all $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(INSTALL_BIN) $(REDIS_INSTALL) $(REDIS_BENCHMARK_NAME) $(INSTALL_BIN) $(REDIS_INSTALL) $(REDIS_CLI_NAME) $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_CHECK_RDB_NAME) $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_CHECK_AOF_NAME) $(INSTALL_BIN) + @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_CHECK_RDB_NAME) + @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_CHECK_AOF_NAME) @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_SENTINEL_NAME) uninstall: diff --git a/src/acl.cpp b/src/acl.cpp index beec3e409..1a35b5bd9 100644 --- a/src/acl.cpp +++ b/src/acl.cpp @@ -55,6 +55,10 @@ list *UsersToLoad; /* This is a list of users found in the configuration file list *ACLLog; /* Our security log, the user is able to inspect that using the ACL LOG command .*/ +static rax *commandId = NULL; /* Command name to id mapping */ + +static unsigned long nextid = 0; /* Next command id that has not been assigned */ + struct ACLCategoryItem { const char *name; uint64_t flag; @@ -88,18 +92,22 @@ struct ACLUserFlag { const char *name; uint64_t flag; } ACLUserFlags[] = { + /* Note: the order here dictates the emitted order at ACLDescribeUser */ {"on", USER_FLAG_ENABLED}, {"off", USER_FLAG_DISABLED}, {"allkeys", USER_FLAG_ALLKEYS}, + {"allchannels", USER_FLAG_ALLCHANNELS}, {"allcommands", USER_FLAG_ALLCOMMANDS}, {"nopass", USER_FLAG_NOPASS}, + {"skip-sanitize-payload", USER_FLAG_SANITIZE_PAYLOAD_SKIP}, + {"sanitize-payload", USER_FLAG_SANITIZE_PAYLOAD}, {NULL,0} /* Terminator. */ }; void ACLResetSubcommandsForCommand(user *u, unsigned long id); void ACLResetSubcommands(user *u); void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub); -void ACLFreeLogEntry(struct ACLLogEntry *le); +void ACLFreeLogEntry(const void *le); /* The length of the string representation of a hashed password. */ #define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2 @@ -169,15 +177,15 @@ sds ACLHashPassword(unsigned char *cleartext, size_t len) { return sdsnewlen(hex,HASH_PASSWORD_LEN); } -/* Given a hash and the hash length, returns C_OK if it is a valid password +/* Given a hash and the hash length, returns C_OK if it is a valid password * hash, or C_ERR otherwise. */ int ACLCheckPasswordHash(unsigned char *hash, int hashlen) { if (hashlen != HASH_PASSWORD_LEN) { - return C_ERR; + return C_ERR; } - + /* Password hashes can only be characters that represent - * hexadecimal values, which are numbers and lowercase + * hexadecimal values, which are numbers and lowercase * characters 'a' through 'f'. */ for(int i = 0; i < HASH_PASSWORD_LEN; i++) { char c = hash[i]; @@ -240,16 +248,20 @@ user *ACLCreateUser(const char *name, size_t namelen) { if (raxFind(Users,(unsigned char*)name,namelen) != raxNotFound) return NULL; user *u = (user*)zmalloc(sizeof(*u), MALLOC_LOCAL); u->name = sdsnewlen(name,namelen); - u->flags = USER_FLAG_DISABLED; + u->flags = USER_FLAG_DISABLED | g_pserver->acl_pubusub_default; u->allowed_subcommands = NULL; u->passwords = listCreate(); u->patterns = listCreate(); + u->channels = listCreate(); listSetMatchMethod(u->passwords,ACLListMatchSds); listSetFreeMethod(u->passwords,ACLListFreeSds); listSetDupMethod(u->passwords,ACLListDupSds); listSetMatchMethod(u->patterns,ACLListMatchSds); listSetFreeMethod(u->patterns,ACLListFreeSds); listSetDupMethod(u->patterns,ACLListDupSds); + listSetMatchMethod(u->channels,ACLListMatchSds); + listSetFreeMethod(u->channels,ACLListFreeSds); + listSetDupMethod(u->channels,ACLListDupSds); memset(u->allowed_commands,0,sizeof(u->allowed_commands)); raxInsert(Users,(unsigned char*)name,namelen,u,NULL); return u; @@ -278,6 +290,7 @@ void ACLFreeUser(user *u) { sdsfree(u->name); listRelease(u->passwords); listRelease(u->patterns); + listRelease(u->channels); ACLResetSubcommands(u); zfree(u); } @@ -291,14 +304,14 @@ void ACLFreeUserAndKillClients(user *u) { listRewind(g_pserver->clients,&li); while ((ln = listNext(&li)) != NULL) { client *c = (client*)listNodeValue(ln); - if (c->puser == u) { - /* We'll free the conenction asynchronously, so + if (c->user == u) { + /* We'll free the connection asynchronously, so * in theory to set a different user is not needed. * However if there are bugs in Redis, soon or later * this may result in some security hole: it's much * more defensive to set the default user and put * it in non authenticated mode. */ - c->puser = DefaultUser; + c->user = DefaultUser; c->authenticated = 0; /* We will write replies to this client later, so we can't * close it directly even if async. */ @@ -318,8 +331,10 @@ void ACLFreeUserAndKillClients(user *u) { void ACLCopyUser(user *dst, user *src) { listRelease(dst->passwords); listRelease(dst->patterns); + listRelease(dst->channels); dst->passwords = listDup(src->passwords); dst->patterns = listDup(src->patterns); + dst->channels = listDup(src->channels); memcpy(dst->allowed_commands,src->allowed_commands, sizeof(dst->allowed_commands)); dst->flags = src->flags; @@ -601,9 +616,10 @@ sds ACLDescribeUser(user *u) { /* Flags. */ for (int j = 0; ACLUserFlags[j].flag; j++) { - /* Skip the allcommands and allkeys flags because they'll be emitted - * later as ~* and +@all. */ + /* Skip the allcommands, allkeys and allchannels flags because they'll + * be emitted later as +@all, ~* and &*. */ if (ACLUserFlags[j].flag == USER_FLAG_ALLKEYS || + ACLUserFlags[j].flag == USER_FLAG_ALLCHANNELS || ACLUserFlags[j].flag == USER_FLAG_ALLCOMMANDS) continue; if (u->flags & ACLUserFlags[j].flag) { res = sdscat(res,ACLUserFlags[j].name); @@ -635,6 +651,19 @@ sds ACLDescribeUser(user *u) { } } + /* Pub/sub channel patterns. */ + if (u->flags & USER_FLAG_ALLCHANNELS) { + res = sdscatlen(res,"&* ",3); + } else { + listRewind(u->channels,&li); + while((ln = listNext(&li))) { + sds thispat = (sds)listNodeValue(ln); + res = sdscatlen(res,"&",1); + res = sdscatsds(res,thispat); + res = sdscatlen(res," ",1); + } + } + /* Command rules. */ sds rules = ACLDescribeUserCommandRules(u); res = sdscatsds(res,rules); @@ -680,7 +709,6 @@ void ACLResetSubcommands(user *u) { u->allowed_subcommands = NULL; } - /* Add a subcommand to the list of subcommands for the user 'u' and * the command id specified. */ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { @@ -741,6 +769,12 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { * It is possible to specify multiple patterns. * allkeys Alias for ~* * resetkeys Flush the list of allowed keys patterns. + * & Add a pattern of channels that can be mentioned as part of + * Pub/Sub commands. For instance &* allows all the channels. The + * pattern is a glob-style pattern like the one of PSUBSCRIBE. + * It is possible to specify multiple patterns. + * allchannels Alias for &* + * resetchannels Flush the list of allowed keys patterns. * > Add this password to the list of valid password for the user. * For example >mypass will add "mypass" to the list. * This directive clears the "nopass" flag (see later). @@ -779,14 +813,14 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { * * When an error is returned, errno is set to the following values: * - * EINVAL: The specified opcode is not understood or the key pattern is + * EINVAL: The specified opcode is not understood or the key/channel pattern is * invalid (contains non allowed characters). * ENOENT: The command name or command category provided with + or - is not * known. - * EBUSY: The subcommand you want to add is about a command that is currently - * fully added. * EEXIST: You are adding a key pattern after "*" was already added. This is * almost surely an error on the user side. + * EISDIR: You are adding a channel pattern after "*" was already added. This is + * almost surely an error on the user side. * ENODEV: The password you are trying to remove from the user does not exist. * EBADMSG: The hash you are trying to add is not a valid hash. */ @@ -799,6 +833,12 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { } else if (!strcasecmp(op,"off")) { u->flags |= USER_FLAG_DISABLED; u->flags &= ~USER_FLAG_ENABLED; + } else if (!strcasecmp(op,"skip-sanitize-payload")) { + u->flags |= USER_FLAG_SANITIZE_PAYLOAD_SKIP; + u->flags &= ~USER_FLAG_SANITIZE_PAYLOAD; + } else if (!strcasecmp(op,"sanitize-payload")) { + u->flags &= ~USER_FLAG_SANITIZE_PAYLOAD_SKIP; + u->flags |= USER_FLAG_SANITIZE_PAYLOAD; } else if (!strcasecmp(op,"allkeys") || !strcasecmp(op,"~*")) { @@ -807,6 +847,14 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { } else if (!strcasecmp(op,"resetkeys")) { u->flags &= ~USER_FLAG_ALLKEYS; listEmpty(u->patterns); + } else if (!strcasecmp(op,"allchannels") || + !strcasecmp(op,"&*")) + { + u->flags |= USER_FLAG_ALLCHANNELS; + listEmpty(u->channels); + } else if (!strcasecmp(op,"resetchannels")) { + u->flags &= ~USER_FLAG_ALLCHANNELS; + listEmpty(u->channels); } else if (!strcasecmp(op,"allcommands") || !strcasecmp(op,"+@all")) { @@ -874,12 +922,29 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { } sds newpat = sdsnewlen(op+1,oplen-1); listNode *ln = listSearchKey(u->patterns,newpat); - /* Avoid re-adding the same pattern multiple times. */ + /* Avoid re-adding the same key pattern multiple times. */ if (ln == NULL) listAddNodeTail(u->patterns,newpat); else sdsfree(newpat); u->flags &= ~USER_FLAG_ALLKEYS; + } else if (op[0] == '&') { + if (u->flags & USER_FLAG_ALLCHANNELS) { + errno = EISDIR; + return C_ERR; + } + if (ACLStringHasSpaces(op+1,oplen-1)) { + errno = EINVAL; + return C_ERR; + } + sds newpat = sdsnewlen(op+1,oplen-1); + listNode *ln = listSearchKey(u->channels,newpat); + /* Avoid re-adding the same channel pattern multiple times. */ + if (ln == NULL) + listAddNodeTail(u->channels,newpat); + else + sdsfree(newpat); + u->flags &= ~USER_FLAG_ALLCHANNELS; } else if (op[0] == '+' && op[1] != '@') { if (strchr(op,'|') == NULL) { if (ACLLookupCommand(op+1) == NULL) { @@ -912,22 +977,12 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { return C_ERR; } - /* The command should not be set right now in the command - * bitmap, because adding a subcommand of a fully added - * command is probably an error on the user side. */ unsigned long id = ACLGetCommandID(copy); - if (ACLGetUserCommandBit(u,id) == 1) { - zfree(copy); - errno = EBUSY; - return C_ERR; + /* Add the subcommand to the list of valid ones, if the command is not set. */ + if (ACLGetUserCommandBit(u,id) == 0) { + ACLAddAllowedSubcommand(u,id,sub); } - /* Add the subcommand to the list of valid ones. */ - ACLAddAllowedSubcommand(u,id,sub); - - /* We have to clear the command bit so that we force the - * subcommand check. */ - ACLSetUserCommandBit(u,id,0); zfree(copy); } } else if (op[0] == '-' && op[1] != '@') { @@ -947,7 +1002,9 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { } else if (!strcasecmp(op,"reset")) { serverAssert(ACLSetUser(u,"resetpass",-1) == C_OK); serverAssert(ACLSetUser(u,"resetkeys",-1) == C_OK); + serverAssert(ACLSetUser(u,"resetchannels",-1) == C_OK); serverAssert(ACLSetUser(u,"off",-1) == C_OK); + serverAssert(ACLSetUser(u,"sanitize-payload",-1) == C_OK); serverAssert(ACLSetUser(u,"-@all",-1) == C_OK); } else { errno = EINVAL; @@ -964,15 +1021,16 @@ const char *ACLSetUserStringError(void) { errmsg = "Unknown command or category name in ACL"; else if (errno == EINVAL) errmsg = "Syntax error"; - else if (errno == EBUSY) - errmsg = "Adding a subcommand of a command already fully " - "added is not allowed. Remove the command to start. " - "Example: -DEBUG +DEBUG|DIGEST"; else if (errno == EEXIST) errmsg = "Adding a pattern after the * pattern (or the " "'allkeys' flag) is not valid and does not have any " "effect. Try 'resetkeys' to start with an empty " "list of patterns"; + else if (errno == EISDIR) + errmsg = "Adding a pattern after the * pattern (or the " + "'allchannels' flag) is not valid and does not have any " + "effect. Try 'resetchannels' to start with an empty " + "list of channels"; else if (errno == ENODEV) errmsg = "The password you are trying to remove from the user does " "not exist"; @@ -988,6 +1046,7 @@ void ACLInitDefaultUser(void) { DefaultUser = ACLCreateUser("default",7); ACLSetUser(DefaultUser,"+@all",-1); ACLSetUser(DefaultUser,"~*",-1); + ACLSetUser(DefaultUser,"&*",-1); ACLSetUser(DefaultUser,"on",-1); ACLSetUser(DefaultUser,"nopass",-1); } @@ -998,7 +1057,6 @@ void ACLInit(void) { UsersToLoad = listCreate(); ACLLog = listCreate(); ACLInitDefaultUser(); - g_pserver->requirepass = NULL; /* Only used for backward compatibility. */ } /* Check the username and password pair and return C_OK if they are valid, @@ -1052,7 +1110,7 @@ int ACLCheckUserCredentials(robj *username, robj *password) { int ACLAuthenticateUser(client *c, robj *username, robj *password) { if (ACLCheckUserCredentials(username,password) == C_OK) { c->authenticated = 1; - c->puser = ACLGetUserByName((sds)ptrFromObj(username),sdslen((sds)ptrFromObj(username))); + c->user = ACLGetUserByName((sds)ptrFromObj(username),sdslen((sds)ptrFromObj(username))); moduleNotifyUserChanged(c); return C_OK; } else { @@ -1068,18 +1126,16 @@ int ACLAuthenticateUser(client *c, robj *username, robj *password) { * command name, so that a command retains the same ID in case of modules that * are unloaded and later reloaded. */ unsigned long ACLGetCommandID(const char *cmdname) { - static rax *map = NULL; - static unsigned long nextid = 0; sds lowername = sdsnew(cmdname); sdstolower(lowername); - if (map == NULL) map = raxNew(); - void *id = raxFind(map,(unsigned char*)lowername,sdslen(lowername)); + if (commandId == NULL) commandId = raxNew(); + void *id = raxFind(commandId,(unsigned char*)lowername,sdslen(lowername)); if (id != raxNotFound) { sdsfree(lowername); return (unsigned long)id; } - raxInsert(map,(unsigned char*)lowername,strlen(lowername), + raxInsert(commandId,(unsigned char*)lowername,strlen(lowername), (void*)nextid,NULL); sdsfree(lowername); unsigned long thisid = nextid; @@ -1097,6 +1153,13 @@ unsigned long ACLGetCommandID(const char *cmdname) { return thisid; } +/* Clear command id table and reset nextid to 0. */ +void ACLClearCommandID(void) { + if (commandId) raxFree(commandId); + commandId = NULL; + nextid = 0; +} + /* Return an username by its name, or NULL if the user does not exist. */ user *ACLGetUserByName(const char *name, size_t namelen) { void *myuser = raxFind(Users,(unsigned char*)name,namelen); @@ -1106,7 +1169,7 @@ user *ACLGetUserByName(const char *name, size_t namelen) { /* Check if the command is ready to be executed in the client 'c', already * referenced by c->cmd, and can be executed by this client according to the - * ACLs associated to the client user c->puser. + * ACLs associated to the client user c->user. * * If the user can execute the command ACL_OK is returned, otherwise * ACL_DENIED_CMD or ACL_DENIED_KEY is returned: the first in case the @@ -1114,7 +1177,7 @@ user *ACLGetUserByName(const char *name, size_t namelen) { * command, the second if the command is denied because the user is trying * to access keys that are not among the specified patterns. */ int ACLCheckCommandPerm(client *c, int *keyidxptr) { - user *u = c->puser; + user *u = c->user; uint64_t id = c->cmd->id; /* If there is no associated user, the connection can run anything. */ @@ -1149,7 +1212,7 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) { /* Check if the user can execute commands explicitly touching the keys * mentioned in the command arguments. */ - if (!(c->puser->flags & USER_FLAG_ALLKEYS) && + if (!(c->user->flags & USER_FLAG_ALLKEYS) && (c->cmd->getkeys_proc || c->cmd->firstkey)) { getKeysResult result = GETKEYS_RESULT_INIT; @@ -1187,6 +1250,119 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) { return ACL_OK; } +/* Check if the provided channel is whitelisted by the given allowed channels + * list. Glob-style pattern matching is employed, unless the literal flag is + * set. Returns ACL_OK if access is granted or ACL_DENIED_CHANNEL otherwise. */ +int ACLCheckPubsubChannelPerm(sds channel, list *allowed, int literal) { + listIter li; + listNode *ln; + size_t clen = sdslen(channel); + int match = 0; + + listRewind(allowed,&li); + while((ln = listNext(&li))) { + sds pattern = (sds)listNodeValue(ln); + size_t plen = sdslen(pattern); + + if ((literal && !sdscmp(pattern,channel)) || + (!literal && stringmatchlen(pattern,plen,channel,clen,0))) + { + match = 1; + break; + } + } + if (!match) { + return ACL_DENIED_CHANNEL; + } + return ACL_OK; +} + +/* Check if the user's existing pub/sub clients violate the ACL pub/sub + * permissions specified via the upcoming argument, and kill them if so. */ +void ACLKillPubsubClientsIfNeeded(user *u, list *upcoming) { + listIter li, lpi; + listNode *ln, *lpn; + robj *o; + int kill = 0; + + /* Nothing to kill when the upcoming are a literal super set of the original + * permissions. */ + listRewind(u->channels,&li); + while (!kill && ((ln = listNext(&li)) != NULL)) { + sds pattern = (sds)listNodeValue(ln); + kill = (ACLCheckPubsubChannelPerm(pattern,upcoming,1) == + ACL_DENIED_CHANNEL); + } + if (!kill) return; + + /* Scan all connected clients to find the user's pub/subs. */ + listRewind(g_pserver->clients,&li); + while ((ln = listNext(&li)) != NULL) { + client *c = (client*)listNodeValue(ln); + kill = 0; + + if (c->user == u && getClientType(c) == CLIENT_TYPE_PUBSUB) { + /* Check for pattern violations. */ + listRewind(c->pubsub_patterns,&lpi); + while (!kill && ((lpn = listNext(&lpi)) != NULL)) { + o = (robj*)lpn->value; + kill = (ACLCheckPubsubChannelPerm(szFromObj(o),upcoming,1) == + ACL_DENIED_CHANNEL); + } + /* Check for channel violations. */ + if (!kill) { + dictIterator *di = dictGetIterator(c->pubsub_channels); + dictEntry *de; + while (!kill && ((de = dictNext(di)) != NULL)) { + o = (robj*)dictGetKey(de); + kill = (ACLCheckPubsubChannelPerm(szFromObj(o),upcoming,0) == + ACL_DENIED_CHANNEL); + } + dictReleaseIterator(di); + } + + /* Kill it. */ + if (kill) { + freeClientAsync(c); + } + } + } +} + +/* Check if the pub/sub channels of the command, that's ready to be executed in + * the client 'c', can be executed by this client according to the ACLs channels + * associated to the client user c->user. + * + * idx and count are the index and count of channel arguments from the + * command. The literal argument controls whether the user's ACL channels are + * evaluated as literal values or matched as glob-like patterns. + * + * If the user can execute the command ACL_OK is returned, otherwise + * ACL_DENIED_CHANNEL. */ +int ACLCheckPubsubPerm(client *c, int idx, int count, int literal, int *idxptr) { + user *u = c->user; + + /* If there is no associated user, the connection can run anything. */ + if (u == NULL) return ACL_OK; + + /* Check if the user can access the channels mentioned in the command's + * arguments. */ + if (!(c->user->flags & USER_FLAG_ALLCHANNELS)) { + for (int j = idx; j < idx+count; j++) { + if (ACLCheckPubsubChannelPerm(szFromObj(c->argv[j]),u->channels,literal) + != ACL_OK) { + if (idxptr) *idxptr = j; + return ACL_DENIED_CHANNEL; + } + } + } + + /* If we survived all the above checks, the user can execute the + * command. */ + return ACL_OK; + +} + /* ============================================================================= * ACL loading / saving functions * ==========================================================================*/ @@ -1588,8 +1764,8 @@ int ACLLogMatchEntry(ACLLogEntry *a, ACLLogEntry *b) { } /* Release an ACL log entry. */ -void ACLFreeLogEntry(ACLLogEntry *leptr) { - ACLLogEntry *le = leptr; +void ACLFreeLogEntry(const void *leptr) { + ACLLogEntry *le = (ACLLogEntry*)leptr; sdsfree(le->object); sdsfree(le->username); sdsfree(le->cinfo); @@ -1602,24 +1778,25 @@ void ACLFreeLogEntry(ACLLogEntry *leptr) { * the log entry instead of creating many entries for very similar ACL * rules issues. * - * The keypos argument is only used when the reason is ACL_DENIED_KEY, since - * it allows the function to log the key name that caused the problem. - * Similarly the username is only passed when we failed to authenticate the - * user with AUTH or HELLO, for the ACL_DENIED_AUTH reason. Otherwise - * it will just be NULL. + * The argpos argument is used when the reason is ACL_DENIED_KEY or + * ACL_DENIED_CHANNEL, since it allows the function to log the key or channel + * name that caused the problem. Similarly the username is only passed when we + * failed to authenticate the user with AUTH or HELLO, for the ACL_DENIED_AUTH + * reason. Otherwise it will just be NULL. */ -void addACLLogEntry(client *c, int reason, int keypos, sds username) { +void addACLLogEntry(client *c, int reason, int argpos, sds username) { /* Create a new entry. */ struct ACLLogEntry *le = (ACLLogEntry*)zmalloc(sizeof(*le)); le->count = 1; le->reason = reason; - le->username = sdsdup(reason == ACL_DENIED_AUTH ? username : c->puser->name); + le->username = sdsdup(reason == ACL_DENIED_AUTH ? username : c->user->name); le->ctime = mstime(); switch(reason) { case ACL_DENIED_CMD: le->object = sdsnew(c->cmd->name); break; - case ACL_DENIED_KEY: le->object = sdsnew(szFromObj(c->argv[keypos])); break; - case ACL_DENIED_AUTH: le->object = sdsnew(szFromObj(c->argv[0])); break; + case ACL_DENIED_KEY: le->object = sdsdup(szFromObj(c->argv[argpos])); break; + case ACL_DENIED_CHANNEL: le->object = sdsdup(szFromObj(c->argv[argpos])); break; + case ACL_DENIED_AUTH: le->object = sdsdup(szFromObj(c->argv[0])); break; default: le->object = sdsempty(); } @@ -1666,7 +1843,7 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username) { le->cinfo = NULL; ACLFreeLogEntry(le); } else { - /* Add it to our list of entires. We'll have to trim the list + /* Add it to our list of entries. We'll have to trim the list * to its maximum size. */ listAddNodeHead(ACLLog, le); while(listLength(ACLLog) > g_pserver->acllog_max_len) { @@ -1727,6 +1904,11 @@ void aclCommand(client *c) { } } + /* Existing pub/sub clients authenticated with the user may need to be + * disconnected if (some of) their channel permissions were revoked. */ + if (u && !(tempu->flags & USER_FLAG_ALLCHANNELS)) + ACLKillPubsubClientsIfNeeded(u,tempu->channels); + /* Overwrite the user with the temporary user we modified above. */ if (!u) u = ACLCreateUser(username,sdslen(username)); serverAssert(u != NULL); @@ -1762,7 +1944,7 @@ void aclCommand(client *c) { return; } - addReplyMapLen(c,4); + addReplyMapLen(c,5); /* Flags */ addReplyBulkCString(c,"flags"); @@ -1807,6 +1989,22 @@ void aclCommand(client *c) { addReplyBulkCBuffer(c,thispat,sdslen(thispat)); } } + + /* Pub/sub patterns */ + addReplyBulkCString(c,"channels"); + if (u->flags & USER_FLAG_ALLCHANNELS) { + addReplyArrayLen(c,1); + addReplyBulkCBuffer(c,"*",1); + } else { + addReplyArrayLen(c,listLength(u->channels)); + listIter li; + listNode *ln; + listRewind(u->channels,&li); + while((ln = listNext(&li))) { + sds thispat = (sds)listNodeValue(ln); + addReplyBulkCBuffer(c,thispat,sdslen(thispat)); + } + } } else if ((!strcasecmp(sub,"list") || !strcasecmp(sub,"users")) && c->argc == 2) { @@ -1832,8 +2030,8 @@ void aclCommand(client *c) { } raxStop(&ri); } else if (!strcasecmp(sub,"whoami") && c->argc == 2) { - if (c->puser != NULL) { - addReplyBulkCBuffer(c,c->puser->name,sdslen(c->puser->name)); + if (c->user != NULL) { + addReplyBulkCBuffer(c,c->user->name,sdslen(c->user->name)); } else { addReplyNull(c); } @@ -1911,7 +2109,7 @@ void aclCommand(client *c) { * the "RESET" command in order to flush the old entries. */ if (c->argc == 3) { if (!strcasecmp(szFromObj(c->argv[2]),"reset")) { - listSetFreeMethod(ACLLog,(void(*)(const void*))ACLFreeLogEntry); + listSetFreeMethod(ACLLog,ACLFreeLogEntry); listEmpty(ACLLog); listSetFreeMethod(ACLLog,NULL); addReply(c,shared.ok); @@ -1940,10 +2138,11 @@ void aclCommand(client *c) { addReplyLongLong(c,le->count); addReplyBulkCString(c,"reason"); - const char *reasonstr = "INVALID_REASON"; + const char *reasonstr; switch(le->reason) { case ACL_DENIED_CMD: reasonstr="command"; break; case ACL_DENIED_KEY: reasonstr="key"; break; + case ACL_DENIED_CHANNEL: reasonstr="channel"; break; case ACL_DENIED_AUTH: reasonstr="auth"; break; default: reasonstr="unknown"; } @@ -1971,18 +2170,30 @@ void aclCommand(client *c) { } } else if (c->argc == 2 && !strcasecmp(sub,"help")) { const char *help[] = { -"LOAD -- Reload users from the ACL file.", -"SAVE -- Save the current config to the ACL file.", -"LIST -- Show user details in config file format.", -"USERS -- List all the registered usernames.", -"SETUSER [attribs ...] -- Create or modify a user.", -"GETUSER -- Get the user details.", -"DELUSER [...] -- Delete a list of users.", -"CAT -- List available categories.", -"CAT -- List commands inside category.", -"GENPASS [] -- Generate a secure user password.", -"WHOAMI -- Return the current connection username.", -"LOG [ | RESET] -- Show the ACL log entries.", +"CAT []", +" List all commands that belong to , or all command categories", +" when no category is specified.", +"DELUSER [ ...]", +" Delete a list of users.", +"GETUSER ", +" Get the user's details.", +"GENPASS []", +" Generate a secure 256-bit user password. The optional `bits` argument can", +" be used to specify a different size.", +"LIST", +" Show users details in config file format.", +"LOAD", +" Reload users from the ACL file.", +"LOG [ | RESET]", +" Show the ACL log entries.", +"SAVE", +" Save the current config to the ACL file.", +"SETUSER [ ...]", +" Create or modify a user with the specified attributes.", +"USERS", +" List all the registered usernames.", +"WHOAMI", +" Return the current connection username.", NULL }; addReplyHelp(c,help); @@ -2011,7 +2222,7 @@ void addReplyCommandCategories(client *c, struct redisCommand *cmd) { void authCommand(client *c) { /* Only two or three argument forms are allowed. */ if (c->argc > 3) { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } @@ -2028,7 +2239,7 @@ void authCommand(client *c) { return; } - username = createStringObject("default",7); + username = shared.default_username; password = c->argv[1]; } else { username = c->argv[1]; @@ -2038,11 +2249,19 @@ void authCommand(client *c) { if (ACLAuthenticateUser(c,username,password) == C_OK) { addReply(c,shared.ok); } else { - addReplyError(c,"-WRONGPASS invalid username-password pair"); + addReplyError(c,"-WRONGPASS invalid username-password pair or user is disabled."); } - - /* Free the "default" string object we created for the two - * arguments form. */ - if (c->argc == 2) decrRefCount(username); } +/* Set the password for the "default" ACL user. This implements supports for + * requirepass config, so passing in NULL will set the user to be nopass. */ +void ACLUpdateDefaultUserPassword(sds password) { + ACLSetUser(DefaultUser,"resetpass",-1); + if (password) { + sds aclop = sdscatlen(sdsnew(">"), password, sdslen(password)); + ACLSetUser(DefaultUser,aclop,sdslen(aclop)); + sdsfree(aclop); + } else { + ACLSetUser(DefaultUser,"nopass",-1); + } +} diff --git a/src/adlist.c b/src/adlist.c index 6d5d77fb3..1adb0bcb0 100644 --- a/src/adlist.c +++ b/src/adlist.c @@ -360,15 +360,16 @@ void listRotateHeadToTail(list *list) { /* Add all the elements of the list 'o' at the end of the * list 'l'. The list 'other' remains empty but otherwise valid. */ void listJoin(list *l, list *o) { - if (o->head) - o->head->prev = l->tail; + if (o->len == 0) return; + + o->head->prev = l->tail; if (l->tail) l->tail->next = o->head; else l->head = o->head; - if (o->tail) l->tail = o->tail; + l->tail = o->tail; l->len += o->len; /* Setup other as an empty list. */ diff --git a/src/ae.cpp b/src/ae.cpp index 16a2ddb64..abe6b5bb7 100644 --- a/src/ae.cpp +++ b/src/ae.cpp @@ -30,6 +30,10 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#include "ae.h" +#include "anet.h" +#include "fastlock.h" + #include #include #include @@ -271,12 +275,13 @@ aeEventLoop *aeCreateEventLoop(int setsize) { aeEventLoop *eventLoop; int i; + monotonicInit(); /* just in case the calling app didn't initialize */ + if ((eventLoop = (aeEventLoop*)zmalloc(sizeof(*eventLoop), MALLOC_LOCAL)) == NULL) goto err; eventLoop->events = (aeFileEvent*)zmalloc(sizeof(aeFileEvent)*setsize, MALLOC_LOCAL); eventLoop->fired = (aeFiredEvent*)zmalloc(sizeof(aeFiredEvent)*setsize, MALLOC_LOCAL); if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err; eventLoop->setsize = setsize; - eventLoop->lastTime = time(NULL); eventLoop->timeEventHead = NULL; eventLoop->timeEventNextId = 0; eventLoop->stop = 0; @@ -448,29 +453,6 @@ extern "C" int aeGetFileEvents(aeEventLoop *eventLoop, int fd) { return fe->mask; } -static void aeGetTime(long *seconds, long *milliseconds) -{ - struct timeval tv; - - gettimeofday(&tv, NULL); - *seconds = tv.tv_sec; - *milliseconds = tv.tv_usec/1000; -} - -static void aeAddMillisecondsToNow(long long milliseconds, long *sec, long *ms) { - long cur_sec, cur_ms, when_sec, when_ms; - - aeGetTime(&cur_sec, &cur_ms); - when_sec = cur_sec + milliseconds/1000; - when_ms = cur_ms + milliseconds%1000; - if (when_ms >= 1000) { - when_sec ++; - when_ms -= 1000; - } - *sec = when_sec; - *ms = when_ms; -} - extern "C" long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, aeTimeProc *proc, void *clientData, aeEventFinalizerProc *finalizerProc) @@ -482,7 +464,7 @@ extern "C" long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long millise te = (aeTimeEvent*)zmalloc(sizeof(*te), MALLOC_LOCAL); if (te == NULL) return AE_ERR; te->id = id; - aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms); + te->when = getMonotonicUs() + milliseconds * 1000; te->timeProc = proc; te->finalizerProc = finalizerProc; te->clientData = clientData; @@ -509,10 +491,8 @@ extern "C" int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id) return AE_ERR; /* NO event with the specified ID found */ } -/* Search the first timer to fire. - * This operation is useful to know how many time the select can be - * put in sleep without to delay any event. - * If there are no timers NULL is returned. +/* How many milliseconds until the first timer should fire. + * If there are no timers, -1 is returned. * * Note that's O(N) since time events are unsorted. * Possible optimizations (not needed by Redis so far, but...): @@ -520,20 +500,20 @@ extern "C" int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id) * Much better but still insertion or deletion of timers is O(N). * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)). */ -static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop) -{ - serverAssert(g_eventLoopThisThread == NULL || g_eventLoopThisThread == eventLoop); +static long msUntilEarliestTimer(aeEventLoop *eventLoop) { aeTimeEvent *te = eventLoop->timeEventHead; - aeTimeEvent *nearest = NULL; + if (te == NULL) return -1; - while(te) { - if (!nearest || te->when_sec < nearest->when_sec || - (te->when_sec == nearest->when_sec && - te->when_ms < nearest->when_ms)) - nearest = te; + aeTimeEvent *earliest = NULL; + while (te) { + if (!earliest || te->when < earliest->when) + earliest = te; te = te->next; } - return nearest; + + monotime now = getMonotonicUs(); + return (now >= earliest->when) + ? 0 : (long)((earliest->when - now) / 1000); } /* Process time events */ @@ -542,29 +522,11 @@ static int processTimeEvents(aeEventLoop *eventLoop) { int processed = 0; aeTimeEvent *te; long long maxId; - time_t now = time(NULL); - - /* If the system clock is moved to the future, and then set back to the - * right value, time events may be delayed in a random way. Often this - * means that scheduled operations will not be performed soon enough. - * - * Here we try to detect system clock skews, and force all the time - * events to be processed ASAP when this happens: the idea is that - * processing events earlier is less dangerous than delaying them - * indefinitely, and practice suggests it is. */ - if (now < eventLoop->lastTime) { - te = eventLoop->timeEventHead; - while(te) { - te->when_sec = 0; - te = te->next; - } - } - eventLoop->lastTime = now; te = eventLoop->timeEventHead; maxId = eventLoop->timeEventNextId-1; + monotime now = getMonotonicUs(); while(te) { - long now_sec, now_ms; long long id; /* Remove events scheduled for deletion. */ @@ -586,6 +548,7 @@ static int processTimeEvents(aeEventLoop *eventLoop) { if (te->finalizerProc) { if (!ulock.owns_lock()) ulock.lock(); te->finalizerProc(eventLoop, te->clientData); + now = getMonotonicUs(); } zfree(te); te = next; @@ -601,10 +564,8 @@ static int processTimeEvents(aeEventLoop *eventLoop) { te = te->next; continue; } - aeGetTime(&now_sec, &now_ms); - if (now_sec > te->when_sec || - (now_sec == te->when_sec && now_ms >= te->when_ms)) - { + + if (te->when <= now) { if (!ulock.owns_lock()) ulock.lock(); int retval; @@ -613,8 +574,9 @@ static int processTimeEvents(aeEventLoop *eventLoop) { retval = te->timeProc(eventLoop, id, te->clientData); te->refcount--; processed++; + now = getMonotonicUs(); if (retval != AE_NOMORE) { - aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms); + te->when = now + retval * 1000; } else { te->id = AE_DELETED_EVENT_ID; } @@ -704,37 +666,23 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) /* Nothing to do? return ASAP */ if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0; - /* Note that we want call select() even if there are no + /* Note that we want to call select() even if there are no * file events to process as long as we want to process time * events, in order to sleep until the next time event is ready * to fire. */ if (eventLoop->maxfd != -1 || ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) { int j; - aeTimeEvent *shortest = NULL; struct timeval tv, *tvp; + long msUntilTimer = -1; if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT)) - shortest = aeSearchNearestTimer(eventLoop); - if (shortest) { - long now_sec, now_ms; + msUntilTimer = msUntilEarliestTimer(eventLoop); - aeGetTime(&now_sec, &now_ms); + if (msUntilTimer >= 0) { + tv.tv_sec = msUntilTimer / 1000; + tv.tv_usec = (msUntilTimer % 1000) * 1000; tvp = &tv; - - /* How many milliseconds we need to wait for the next - * time event to fire? */ - long long ms = - (shortest->when_sec - now_sec)*1000 + - shortest->when_ms - now_ms; - - if (ms > 0) { - tvp->tv_sec = ms/1000; - tvp->tv_usec = (ms % 1000)*1000; - } else { - tvp->tv_sec = 0; - tvp->tv_usec = 0; - } } else { /* If we have to check for events but need to return * ASAP because of AE_DONT_WAIT we need to set the timeout @@ -816,14 +764,8 @@ void aeMain(aeEventLoop *eventLoop) { eventLoop->stop = 0; g_eventLoopThisThread = eventLoop; while (!eventLoop->stop) { - if (eventLoop->beforesleep != NULL) { - std::unique_lock ulock(g_lock, std::defer_lock); - if (!(eventLoop->beforesleepFlags & AE_SLEEP_THREADSAFE)) - ulock.lock(); - eventLoop->beforesleep(eventLoop); - } serverAssert(!aeThreadOwnsLock()); // we should have relinquished it after processing - aeProcessEvents(eventLoop, AE_ALL_EVENTS|AE_CALL_AFTER_SLEEP); + aeProcessEvents(eventLoop, AE_ALL_EVENTS|AE_CALL_BEFORE_SLEEP|AE_CALL_AFTER_SLEEP); serverAssert(!aeThreadOwnsLock()); // we should have relinquished it after processing } } diff --git a/src/ae.h b/src/ae.h index 3a240877d..780083da8 100644 --- a/src/ae.h +++ b/src/ae.h @@ -36,7 +36,7 @@ #ifdef __cplusplus #include #endif -#include +#include "monotonic.h" #include "fastlock.h" #ifdef __cplusplus @@ -91,8 +91,7 @@ typedef struct aeFileEvent { /* Time event structure */ typedef struct aeTimeEvent { long long id; /* time event identifier. */ - long when_sec; /* seconds */ - long when_ms; /* milliseconds */ + monotime when; aeTimeProc *timeProc; aeEventFinalizerProc *finalizerProc; void *clientData; @@ -113,7 +112,6 @@ typedef struct aeEventLoop { int maxfd; /* highest file descriptor currently registered */ int setsize; /* max number of file descriptors tracked */ long long timeEventNextId; - time_t lastTime; /* Used to detect system clock skew */ aeFileEvent *events; /* Registered events */ aeFiredEvent *fired; /* Fired events */ aeTimeEvent *timeEventHead; diff --git a/src/ae_epoll.cpp b/src/ae_epoll.cpp index 452fc49ae..67c512063 100644 --- a/src/ae_epoll.cpp +++ b/src/ae_epoll.cpp @@ -51,6 +51,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { zfree(state); return -1; } + anetCloexec(state->epfd); eventLoop->apidata = state; return 0; } diff --git a/src/ae_evport.c b/src/ae_evport.c index 744e1a6bb..de02093ac 100644 --- a/src/ae_evport.c +++ b/src/ae_evport.c @@ -67,7 +67,7 @@ static int evport_debug = 0; typedef struct aeApiState { int portfd; /* event port */ - int npending; /* # of pending fds */ + uint_t npending; /* # of pending fds */ int pending_fds[MAX_EVENT_BATCHSZ]; /* pending fds */ int pending_masks[MAX_EVENT_BATCHSZ]; /* pending fds' masks */ } aeApiState; @@ -82,6 +82,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { zfree(state); return -1; } + anetCloexec(state->portfd); state->npending = 0; @@ -95,6 +96,8 @@ static int aeApiCreate(aeEventLoop *eventLoop) { } static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + (void) eventLoop; + (void) setsize; /* Nothing to resize here. */ return 0; } @@ -107,7 +110,7 @@ static void aeApiFree(aeEventLoop *eventLoop) { } static int aeApiLookupPending(aeApiState *state, int fd) { - int i; + uint_t i; for (i = 0; i < state->npending; i++) { if (state->pending_fds[i] == fd) @@ -243,7 +246,7 @@ static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { aeApiState *state = eventLoop->apidata; struct timespec timeout, *tsp; - int mask, i; + uint_t mask, i; uint_t nevents; port_event_t event[MAX_EVENT_BATCHSZ]; diff --git a/src/ae_kqueue.c b/src/ae_kqueue.c index 5c83f6464..c6e9cf3fa 100644 --- a/src/ae_kqueue.c +++ b/src/ae_kqueue.c @@ -53,6 +53,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { zfree(state); return -1; } + anetCloexec(state->kqfd); eventLoop->apidata = state; return 0; } diff --git a/src/anet.c b/src/anet.c index 8b8b6ae33..dcd0a1347 100644 --- a/src/anet.c +++ b/src/anet.c @@ -69,6 +69,11 @@ int anetSetBlock(char *err, int fd, int non_block) { return ANET_ERR; } + /* Check if this flag has been set or unset, if so, + * then there is no need to call fcntl to set/unset it again. */ + if (!!(flags & O_NONBLOCK) == !!non_block) + return ANET_OK; + if (non_block) flags |= O_NONBLOCK; else @@ -89,6 +94,29 @@ int anetBlock(char *err, int fd) { return anetSetBlock(err,fd,0); } +/* Enable the FD_CLOEXEC on the given fd to avoid fd leaks. + * This function should be invoked for fd's on specific places + * where fork + execve system calls are called. */ +int anetCloexec(int fd) { + int r; + int flags; + + do { + r = fcntl(fd, F_GETFD); + } while (r == -1 && errno == EINTR); + + if (r == -1 || (r & FD_CLOEXEC)) + return r; + + flags = r | FD_CLOEXEC; + + do { + r = fcntl(fd, F_SETFD, flags); + } while (r == -1 && errno == EINTR); + + return r; +} + /* Set TCP keep alive option to detect dead peers. The interval option * is only used for Linux as we are using Linux-specific APIs to set * the probe send time, interval, and count. */ @@ -207,14 +235,13 @@ int anetRecvTimeout(char *err, int fd, long long ms) { return ANET_OK; } -/* anetGenericResolve() is called by anetResolve() and anetResolveIP() to - * do the actual work. It resolves the hostname "host" and set the string - * representation of the IP address into the buffer pointed by "ipbuf". +/* Resolve the hostname "host" and set the string representation of the + * IP address into the buffer pointed by "ipbuf". * * If flags is set to ANET_IP_ONLY the function only resolves hostnames * that are actually already IPv4 or IPv6 addresses. This turns the function * into a validating / normalizing function. */ -int anetGenericResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, int flags) { struct addrinfo hints, *info; @@ -241,14 +268,6 @@ int anetGenericResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, return ANET_OK; } -int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len) { - return anetGenericResolve(err,host,ipbuf,ipbuf_len,ANET_NONE); -} - -int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len) { - return anetGenericResolve(err,host,ipbuf,ipbuf_len,ANET_IP_ONLY); -} - static int anetSetReuseAddr(char *err, int fd) { int yes = 1; /* Make sure connection-intensive things like the redis benchmark @@ -484,13 +503,12 @@ static int anetV6Only(char *err, int s) { int yes = 1; if (setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,&yes,sizeof(yes)) == -1) { anetSetError(err, "setsockopt: %s", strerror(errno)); - close(s); return ANET_ERR; } return ANET_OK; } -static int _anetTcpServer(char *err, int port, char *bindaddr, int af, int backlog, int fReusePort, int fFirstListen) +static int _anetTcpServer(char *err, int port, const char *bindaddr, int af, int backlog, int fReusePort, int fFirstListen) { int s = -1, rv; char _port[6]; /* strlen("65535") */ @@ -501,6 +519,10 @@ static int _anetTcpServer(char *err, int port, char *bindaddr, int af, int backl hints.ai_family = af; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE; /* No effect if bindaddr != NULL */ + if (bindaddr && !strcmp("*", bindaddr)) + bindaddr = NULL; + if (af == AF_INET6 && bindaddr && !strcmp("::*", bindaddr)) + bindaddr = NULL; if ((rv = getaddrinfo(bindaddr,_port,&hints,&servinfo)) != 0) { anetSetError(err, "%s", gai_strerror(rv)); @@ -533,12 +555,12 @@ end: return s; } -int anetTcpServer(char *err, int port, char *bindaddr, int backlog, int fReusePort, int fFirstListen) +int anetTcpServer(char *err, int port, const char *bindaddr, int backlog, int fReusePort, int fFirstListen) { return _anetTcpServer(err, port, bindaddr, AF_INET, backlog, fReusePort, fFirstListen); } -int anetTcp6Server(char *err, int port, char *bindaddr, int backlog, int fReusePort, int fFirstListen) +int anetTcp6Server(char *err, int port, const char *bindaddr, int backlog, int fReusePort, int fFirstListen) { return _anetTcpServer(err, port, bindaddr, AF_INET6, backlog, fReusePort, fFirstListen); } @@ -607,11 +629,15 @@ int anetUnixAccept(char *err, int s) { return fd; } -int anetPeerToString(int fd, char *ip, size_t ip_len, int *port) { +int anetFdToString(int fd, char *ip, size_t ip_len, int *port, int fd_to_str_type) { struct sockaddr_storage sa; socklen_t salen = sizeof(sa); - if (getpeername(fd,(struct sockaddr*)&sa,&salen) == -1) goto error; + if (fd_to_str_type == FD_TO_PEER_NAME) { + if (getpeername(fd, (struct sockaddr *)&sa, &salen) == -1) goto error; + } else { + if (getsockname(fd, (struct sockaddr *)&sa, &salen) == -1) goto error; + } if (ip_len == 0) goto error; if (sa.ss_family == AF_INET) { @@ -623,7 +649,7 @@ int anetPeerToString(int fd, char *ip, size_t ip_len, int *port) { if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len); if (port) *port = ntohs(s->sin6_port); } else if (sa.ss_family == AF_UNIX) { - if (ip) strncpy(ip,"/unixsocket",ip_len); + if (ip) snprintf(ip, ip_len, "/unixsocket"); if (port) *port = 0; } else { goto error; @@ -651,41 +677,11 @@ int anetFormatAddr(char *buf, size_t buf_len, char *ip, int port) { "[%s]:%d" : "%s:%d", ip, port); } -/* Like anetFormatAddr() but extract ip and port from the socket's peer. */ -int anetFormatPeer(int fd, char *buf, size_t buf_len) { +/* Like anetFormatAddr() but extract ip and port from the socket's peer/sockname. */ +int anetFormatFdAddr(int fd, char *buf, size_t buf_len, int fd_to_str_type) { char ip[INET6_ADDRSTRLEN]; int port; - anetPeerToString(fd,ip,sizeof(ip),&port); + anetFdToString(fd,ip,sizeof(ip),&port,fd_to_str_type); return anetFormatAddr(buf, buf_len, ip, port); } - -int anetSockName(int fd, char *ip, size_t ip_len, int *port) { - struct sockaddr_storage sa; - socklen_t salen = sizeof(sa); - - if (getsockname(fd,(struct sockaddr*)&sa,&salen) == -1) { - if (port) *port = 0; - ip[0] = '?'; - ip[1] = '\0'; - return -1; - } - if (sa.ss_family == AF_INET) { - struct sockaddr_in *s = (struct sockaddr_in *)&sa; - if (ip) inet_ntop(AF_INET,(void*)&(s->sin_addr),ip,ip_len); - if (port) *port = ntohs(s->sin_port); - } else { - struct sockaddr_in6 *s = (struct sockaddr_in6 *)&sa; - if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len); - if (port) *port = ntohs(s->sin6_port); - } - return 0; -} - -int anetFormatSock(int fd, char *fmt, size_t fmt_len) { - char ip[INET6_ADDRSTRLEN]; - int port; - - anetSockName(fd,ip,sizeof(ip),&port); - return anetFormatAddr(fmt, fmt_len, ip, port); -} diff --git a/src/anet.h b/src/anet.h index b31a38ad6..aaabc9105 100644 --- a/src/anet.h +++ b/src/anet.h @@ -53,6 +53,10 @@ extern "C" { #undef ip_len #endif +/* FD to address string conversion types */ +#define FD_TO_PEER_NAME 0 +#define FD_TO_SOCK_NAME 1 + int anetTcpConnect(char *err, const char *addr, int port); int anetTcpNonBlockConnect(char *err, const char *addr, int port); int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, const char *source_addr); @@ -60,27 +64,25 @@ int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, int anetUnixConnect(char *err, const char *path); int anetUnixNonBlockConnect(char *err, const char *path); int anetRead(int fd, char *buf, int count); -int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len); -int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len); -int anetTcpServer(char *err, int port, char *bindaddr, int backlog, int fReusePort, int fFirstListen); -int anetTcp6Server(char *err, int port, char *bindaddr, int backlog, int fReusePort, int fFirstListen); +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, int flags); +int anetTcpServer(char *err, int port, const char *bindaddr, int backlog, int fReusePort, int fFirstListen); +int anetTcp6Server(char *err, int port, const char *bindaddr, int backlog, int fReusePort, int fFirstListen); int anetUnixServer(char *err, char *path, mode_t perm, int backlog); int anetTcpAccept(char *err, int serversock, char *ip, size_t ip_len, int *port); int anetUnixAccept(char *err, int serversock); int anetWrite(int fd, char *buf, int count); int anetNonBlock(char *err, int fd); int anetBlock(char *err, int fd); +int anetCloexec(int fd); int anetEnableTcpNoDelay(char *err, int fd); int anetDisableTcpNoDelay(char *err, int fd); int anetTcpKeepAlive(char *err, int fd); int anetSendTimeout(char *err, int fd, long long ms); int anetRecvTimeout(char *err, int fd, long long ms); -int anetPeerToString(int fd, char *ip, size_t ip_len, int *port); +int anetFdToString(int fd, char *ip, size_t ip_len, int *port, int fd_to_str_type); int anetKeepAlive(char *err, int fd, int interval); -int anetSockName(int fd, char *ip, size_t ip_len, int *port); int anetFormatAddr(char *fmt, size_t fmt_len, char *ip, int port); -int anetFormatPeer(int fd, char *fmt, size_t fmt_len); -int anetFormatSock(int fd, char *fmt, size_t fmt_len); +int anetFormatFdAddr(int fd, char *buf, size_t buf_len, int fd_to_str_type); #ifdef __cplusplus } diff --git a/src/aof.cpp b/src/aof.cpp index 51af4a4b4..ec1046947 100644 --- a/src/aof.cpp +++ b/src/aof.cpp @@ -221,29 +221,27 @@ int aofFsyncInProgress(void) { /* Starts a background task that performs fsync() against the specified * file descriptor (the one of the AOF file) in another thread. */ void aof_background_fsync(int fd) { - bioCreateBackgroundJob(BIO_AOF_FSYNC,(void*)(long)fd,NULL,NULL); + bioCreateFsyncJob(fd); } /* Kills an AOFRW child process if exists */ void killAppendOnlyChild(void) { int statloc; /* No AOFRW child? return. */ - if (g_pserver->aof_child_pid == -1) return; + if (g_pserver->child_type != CHILD_TYPE_AOF) return; /* Kill AOFRW child, wait for child exit. */ serverLog(LL_NOTICE,"Killing running AOF rewrite child: %ld", - (long) g_pserver->aof_child_pid); - if (kill(g_pserver->aof_child_pid,SIGUSR1) != -1) { - while(wait3(&statloc,0,NULL) != g_pserver->aof_child_pid); + (long) g_pserver->child_pid); + if (kill(g_pserver->child_pid,SIGUSR1) != -1) { + while(wait3(&statloc,0,NULL) != g_pserver->child_pid); } /* Reset the buffer accumulating changes while the child saves. */ aofRewriteBufferReset(); - aofRemoveTempFile(g_pserver->aof_child_pid); - g_pserver->aof_child_pid = -1; + aofRemoveTempFile(g_pserver->child_pid); + resetChildState(); g_pserver->aof_rewrite_time_start = -1; /* Close pipes used for IPC between the two processes. */ aofClosePipes(); - closeChildInfoPipe(); - updateDictResizePolicy(); } /* Called when the user switches from "appendonly yes" to "appendonly no" @@ -252,6 +250,8 @@ void stopAppendOnly(void) { serverAssert(g_pserver->aof_state != AOF_OFF); flushAppendOnlyFile(1); redis_fsync(g_pserver->aof_fd); + g_pserver->aof_fsync_offset = g_pserver->aof_current_size; + g_pserver->aof_last_fsync = g_pserver->unixtime; close(g_pserver->aof_fd); g_pserver->aof_fd = -1; @@ -259,6 +259,8 @@ void stopAppendOnly(void) { g_pserver->aof_state = AOF_OFF; g_pserver->aof_rewrite_scheduled = 0; killAppendOnlyChild(); + sdsfree(g_pserver->aof_buf); + g_pserver->aof_buf = sdsempty(); } /* Called when the user switches from "appendonly no" to "appendonly yes" @@ -280,14 +282,14 @@ int startAppendOnly(void) { strerror(errno)); return C_ERR; } - if (hasActiveChildProcess() && g_pserver->aof_child_pid == -1) { + if (hasActiveChildProcess() && g_pserver->child_type != CHILD_TYPE_AOF) { g_pserver->aof_rewrite_scheduled = 1; serverLog(LL_WARNING,"AOF was enabled but there is already another background operation. An AOF background was scheduled to start when possible."); } else { /* If there is a pending AOF rewrite, we need to switch it off and * start a new one: the old one cannot be reused because it is not * accumulating the AOF buffer. */ - if (g_pserver->aof_child_pid != -1) { + if (g_pserver->child_type == CHILD_TYPE_AOF) { serverLog(LL_WARNING,"AOF was enabled but there is already an AOF rewriting in background. Stopping background AOF and starting a rewrite now."); killAppendOnlyChild(); } @@ -302,6 +304,12 @@ int startAppendOnly(void) { g_pserver->aof_state = AOF_WAIT_REWRITE; g_pserver->aof_last_fsync = g_pserver->unixtime; g_pserver->aof_fd = newfd; + + /* If AOF was in error state, we just ignore it and log the event. */ + if (g_pserver->aof_last_write_status == C_ERR) { + serverLog(LL_WARNING,"AOF reopen, just ignore the last error."); + g_pserver->aof_last_write_status = C_OK; + } return C_OK; } @@ -471,10 +479,11 @@ void flushAppendOnlyFile(int force) { /* Handle the AOF write error. */ if (g_pserver->aof_fsync == AOF_FSYNC_ALWAYS) { - /* We can't recover when the fsync policy is ALWAYS since the - * reply for the client is already in the output buffers, and we - * have the contract with the user that on acknowledged write data - * is synced on disk. */ + /* We can't recover when the fsync policy is ALWAYS since the reply + * for the client is already in the output buffers (both writes and + * reads), and the changes to the db can't be rolled back. Since we + * have a contract with the user that on acknowledged or observed + * writes are is synced on disk, we must exit. */ serverLog(LL_WARNING,"Can't recover from AOF write error when the AOF fsync policy is 'always'. Exiting..."); exit(1); } else { @@ -522,7 +531,14 @@ try_fsync: /* redis_fsync is defined as fdatasync() for Linux in order to avoid * flushing metadata. */ latencyStartMonitor(latency); - redis_fsync(g_pserver->aof_fd); /* Let's try to get this data on the disk */ + /* Let's try to get this data on the disk. To guarantee data safe when + * the AOF fsync policy is 'always', we should exit if failed to fsync + * AOF (see comment next to the exit(1) after write error above). */ + if (redis_fsync(g_pserver->aof_fd) == -1) { + serverLog(LL_WARNING,"Can't persist AOF for fsync error when the " + "AOF fsync policy is 'always': %s. Exiting...", strerror(errno)); + exit(1); + } latencyEndMonitor(latency); latencyAddSampleIfNeeded("aof-fsync-always",latency); g_pserver->aof_fsync_offset = g_pserver->aof_current_size; @@ -594,11 +610,10 @@ sds catAppendOnlyExpireAtCommand(sds buf, struct redisCommand *cmd, robj *key, r } decrRefCount(seconds); - argv[0] = createStringObject("PEXPIREAT",9); + argv[0] = shared.pexpireat; argv[1] = key; argv[2] = createStringObjectFromLongLong(when); buf = catAppendOnlyGenericCommand(buf, 3, argv); - decrRefCount(argv[0]); decrRefCount(argv[2]); return buf; } @@ -653,43 +668,40 @@ sds catAppendOnlyExpireMemberAtCommand(sds buf, struct redisCommand *cmd, robj * } sds catCommandForAofAndActiveReplication(sds buf, struct redisCommand *cmd, robj **argv, int argc) -{ - robj *tmpargv[3]; - +{ if (cmd->proc == expireCommand || cmd->proc == pexpireCommand || cmd->proc == expireatCommand) { /* Translate EXPIRE/PEXPIRE/EXPIREAT into PEXPIREAT */ buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]); - } else if (cmd->proc == setexCommand || cmd->proc == psetexCommand) { - /* Translate SETEX/PSETEX to SET and PEXPIREAT */ - tmpargv[0] = createStringObject("SET",3); - tmpargv[1] = argv[1]; - tmpargv[2] = argv[3]; - buf = catAppendOnlyGenericCommand(buf,3,tmpargv); - decrRefCount(tmpargv[0]); - buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]); } else if (cmd->proc == setCommand && argc > 3) { - int i; - robj *exarg = NULL, *pxarg = NULL; - for (i = 3; i < argc; i ++) { - if (!strcasecmp(szFromObj(argv[i]), "ex")) exarg = argv[i+1]; - if (!strcasecmp(szFromObj(argv[i]), "px")) pxarg = argv[i+1]; + robj *pxarg = NULL; + /* When SET is used with EX/PX argument setGenericCommand propagates them with PX millisecond argument. + * So since the command arguments are re-written there, we can rely here on the index of PX being 3. */ + if (!strcasecmp(szFromObj(argv[3]), "px")) { + pxarg = argv[4]; } - serverAssert(!(exarg && pxarg)); - if (exarg || pxarg) { - /* Translate SET [EX seconds][PX milliseconds] to SET and PEXPIREAT */ - buf = catAppendOnlyGenericCommand(buf,3,argv); - if (exarg) - buf = catAppendOnlyExpireAtCommand(buf,cserver.expireCommand,argv[1], - exarg); - if (pxarg) - buf = catAppendOnlyExpireAtCommand(buf,cserver.pexpireCommand,argv[1], - pxarg); + /* For AOF we convert SET key value relative time in milliseconds to SET key value absolute time in + * millisecond. Whenever the condition is true it implies that original SET has been transformed + * to SET PX with millisecond time argument so we do not need to worry about unit here.*/ + if (pxarg) { + robj *millisecond = getDecodedObject(pxarg); + long long when = strtoll(szFromObj(millisecond),NULL,10); + when += mstime(); + + decrRefCount(millisecond); + + robj *newargs[5]; + newargs[0] = argv[0]; + newargs[1] = argv[1]; + newargs[2] = argv[2]; + newargs[3] = shared.pxat; + newargs[4] = createStringObjectFromLongLong(when); + buf = catAppendOnlyGenericCommand(buf,5,newargs); + decrRefCount(newargs[4]); } else { buf = catAppendOnlyGenericCommand(buf,argc,argv); } - } else if (cmd->proc == expireMemberCommand || cmd->proc == expireMemberAtCommand || - cmd->proc == pexpireMemberAtCommand) { + } else if (cmd->proc == expireMemberCommand || cmd->proc == expireMemberAtCommand || cmd->proc == pexpireMemberAtCommand) { /* Translate subkey expire commands to PEXPIREMEMBERAT */ buf = catAppendOnlyExpireMemberAtCommand(buf, cmd, argv, argc); } else { @@ -728,7 +740,7 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a * accumulate the differences between the child DB and the current one * in a buffer, so that when the child process will do its work we * can append the differences to the new append only file. */ - if (g_pserver->aof_child_pid != -1) + if (g_pserver->child_type == CHILD_TYPE_AOF) aofRewriteBufferAppend((unsigned char*)buf,sdslen(buf)); sdsfree(buf); @@ -752,11 +764,24 @@ struct client *createAOFClient(void) { c->querybuf_peak = 0; c->argc = 0; c->argv = NULL; + c->original_argc = 0; + c->original_argv = NULL; c->argv_len_sum = 0; c->bufpos = 0; - c->flags = 0; c->fPendingAsyncWrite = FALSE; c->fPendingAsyncWriteHandler = FALSE; + + /* + * The AOF client should never be blocked (unlike master + * replication connection). + * This is because blocking the AOF client might cause + * deadlock (because potentially no one will unblock it). + * Also, if the AOF client will be blocked just for + * background processing there is a chance that the + * command execution order will be violated. + */ + c->flags = CLIENT_DENY_BLOCKING; + c->btype = BLOCKED_NONE; /* We set the fake client as a replica waiting for the synchronization * so that Redis will not try to send replies to this client. */ @@ -766,8 +791,9 @@ struct client *createAOFClient(void) { c->obuf_soft_limit_reached_time = 0; c->watched_keys = listCreate(); c->peerid = NULL; + c->sockname = NULL; c->resp = 2; - c->puser = NULL; + c->user = NULL; listSetFreeMethod(c->reply,freeClientReplyValue); listSetDupMethod(c->reply,dupClientReplyValue); fastlock_init(&c->lock, "fake client"); @@ -790,6 +816,7 @@ void freeFakeClient(struct client *c) { listRelease(c->reply); listRelease(c->watched_keys); freeClientMultiState(c); + freeClientOriginalArgv(c); fastlock_unlock(&c->lock); fastlock_free(&c->lock); zfree(c); @@ -951,7 +978,7 @@ int loadAppendOnlyFile(char *filename) { fakeClient->cmd = NULL; if (g_pserver->aof_load_truncated) valid_up_to = ftello(fp); if (g_pserver->key_load_delay) - usleep(g_pserver->key_load_delay); + debugDelay(g_pserver->key_load_delay); } /* This point can only be reached when EOF is reached without errors. @@ -1052,15 +1079,25 @@ int rewriteListObject(rio *r, robj *key, robj *o) { if (count == 0) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; - if (rioWriteBulkString(r,"RPUSH",5) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items) || + !rioWriteBulkString(r,"RPUSH",5) || + !rioWriteBulkObject(r,key)) + { + quicklistReleaseIterator(li); + return 0; + } } if (entry.value) { - if (rioWriteBulkString(r,(char*)entry.value,entry.sz) == 0) return 0; + if (!rioWriteBulkString(r,(char*)entry.value,entry.sz)) { + quicklistReleaseIterator(li); + return 0; + } } else { - if (rioWriteBulkLongLong(r,entry.longval) == 0) return 0; + if (!rioWriteBulkLongLong(r,entry.longval)) { + quicklistReleaseIterator(li); + return 0; + } } if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; @@ -1086,11 +1123,14 @@ int rewriteSetObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; - if (rioWriteBulkString(r,"SADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items) || + !rioWriteBulkString(r,"SADD",4) || + !rioWriteBulkObject(r,key)) + { + return 0; + } } - if (rioWriteBulkLongLong(r,llval) == 0) return 0; + if (!rioWriteBulkLongLong(r,llval)) return 0; if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } @@ -1104,11 +1144,18 @@ int rewriteSetObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; - if (rioWriteBulkString(r,"SADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items) || + !rioWriteBulkString(r,"SADD",4) || + !rioWriteBulkObject(r,key)) + { + dictReleaseIterator(di); + return 0; + } + } + if (!rioWriteBulkString(r,ele,sdslen(ele))) { + dictReleaseIterator(di); + return 0; } - if (rioWriteBulkString(r,ele,sdslen(ele)) == 0) return 0; if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } @@ -1145,15 +1192,18 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0; - if (rioWriteBulkString(r,"ZADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items*2) || + !rioWriteBulkString(r,"ZADD",4) || + !rioWriteBulkObject(r,key)) + { + return 0; + } } - if (rioWriteBulkDouble(r,score) == 0) return 0; + if (!rioWriteBulkDouble(r,score)) return 0; if (vstr != NULL) { - if (rioWriteBulkString(r,(char*)vstr,vlen) == 0) return 0; + if (!rioWriteBulkString(r,(char*)vstr,vlen)) return 0; } else { - if (rioWriteBulkLongLong(r,vll) == 0) return 0; + if (!rioWriteBulkLongLong(r,vll)) return 0; } zzlNext(zl,&eptr,&sptr); if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; @@ -1172,12 +1222,20 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0; - if (rioWriteBulkString(r,"ZADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items*2) || + !rioWriteBulkString(r,"ZADD",4) || + !rioWriteBulkObject(r,key)) + { + dictReleaseIterator(di); + return 0; + } + } + if (!rioWriteBulkDouble(r,*score) || + !rioWriteBulkString(r,ele,sdslen(ele))) + { + dictReleaseIterator(di); + return 0; } - if (rioWriteBulkDouble(r,*score) == 0) return 0; - if (rioWriteBulkString(r,ele,sdslen(ele)) == 0) return 0; if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } @@ -1226,13 +1284,21 @@ int rewriteHashObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0; - if (rioWriteBulkString(r,"HMSET",5) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items*2) || + !rioWriteBulkString(r,"HMSET",5) || + !rioWriteBulkObject(r,key)) + { + hashTypeReleaseIterator(hi); + return 0; + } } - if (rioWriteHashIteratorCursor(r, hi, OBJ_HASH_KEY) == 0) return 0; - if (rioWriteHashIteratorCursor(r, hi, OBJ_HASH_VALUE) == 0) return 0; + if (!rioWriteHashIteratorCursor(r, hi, OBJ_HASH_KEY) || + !rioWriteHashIteratorCursor(r, hi, OBJ_HASH_VALUE)) + { + hashTypeReleaseIterator(hi); + return 0; + } if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } @@ -1278,6 +1344,20 @@ int rioWriteStreamPendingEntry(rio *r, robj *key, const char *groupname, size_t return 1; } +/* Helper for rewriteStreamObject(): emit the XGROUP CREATECONSUMER is + * needed in order to create consumers that do not have any pending entries. + * All this in the context of the specified key and group. */ +int rioWriteStreamEmptyConsumer(rio *r, robj *key, const char *groupname, size_t groupname_len, streamConsumer *consumer) { + /* XGROUP CREATECONSUMER */ + if (rioWriteBulkCount(r,'*',5) == 0) return 0; + if (rioWriteBulkString(r,"XGROUP",6) == 0) return 0; + if (rioWriteBulkString(r,"CREATECONSUMER",14) == 0) return 0; + if (rioWriteBulkObject(r,key) == 0) return 0; + if (rioWriteBulkString(r,groupname,groupname_len) == 0) return 0; + if (rioWriteBulkString(r,consumer->name,sdslen(consumer->name)) == 0) return 0; + return 1; +} + /* Emit the commands needed to rebuild a stream object. * The function returns 0 on error, 1 on success. */ int rewriteStreamObject(rio *r, robj *key, robj *o) { @@ -1366,13 +1446,25 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { } /* Generate XCLAIMs for each consumer that happens to - * have pending entries. Empty consumers have no semantical - * value so they are discarded. */ + * have pending entries. Empty consumers would be generated with + * XGROUP CREATECONSUMER. */ raxIterator ri_cons; raxStart(&ri_cons,group->consumers); raxSeek(&ri_cons,"^",NULL,0); while(raxNext(&ri_cons)) { streamConsumer *consumer = (streamConsumer*)ri_cons.data; + /* If there are no pending entries, just emit XGROUP CREATECONSUMER */ + if (raxSize(consumer->pel) == 0) { + if (rioWriteStreamEmptyConsumer(r,key,(char*)ri.key, + ri.key_len,consumer) == 0) + { + raxStop(&ri_cons); + raxStop(&ri); + streamIteratorStop(&si); + return 0; + } + continue; + } /* For the current consumer, iterate all the PEL entries * to emit the XCLAIM protocol. */ raxIterator ri_pel; @@ -1438,6 +1530,8 @@ int rewriteAppendOnlyFileRio(rio *aof) { dictEntry *de; size_t processed = 0; int j; + long key_count = 0; + long long updated_time = 0; for (j = 0; j < cserver.dbnum; j++) { char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n"; @@ -1509,6 +1603,17 @@ int rewriteAppendOnlyFileRio(rio *aof) { processed = aof->processed_bytes; aofReadDiffFromParent(); } + + /* Update info every 1 second (approximately). + * in order to avoid calling mstime() on each iteration, we will + * check the diff every 1024 keys */ + if ((key_count++ & 1023) == 0) { + long long now = mstime(); + if (now - updated_time >= 1000) { + sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, "AOF rewrite"); + updated_time = now; + } + } } dictReleaseIterator(di); di = NULL; @@ -1535,6 +1640,7 @@ int rewriteAppendOnlyFile(char *filename) { int nodata = 0; mstime_t start = 0; +{ // BEGIN GOTO SCOPED VARIABLES /* Note that we have to use a different temp name here compared to the * one used by rewriteAppendOnlyFileBackground() function. */ snprintf(tmpfile,256,"temp-rewriteaof-%d.aof", (int) getpid()); @@ -1603,8 +1709,31 @@ int rewriteAppendOnlyFile(char *filename) { serverLog(LL_NOTICE, "Concatenating %.2f MB of AOF diff received from parent.", (double) sdslen(g_pserver->aof_child_diff) / (1024*1024)); - if (rioWrite(&aof,g_pserver->aof_child_diff,sdslen(g_pserver->aof_child_diff)) == 0) - goto werr; + + /* Now we write the entire AOF buffer we received from the parent + * via the pipe during the life of this fork child. + * once a second, we'll take a break and send updated COW info to the parent */ + size_t bytes_to_write = sdslen(g_pserver->aof_child_diff); + const char *buf = g_pserver->aof_child_diff; + long long cow_updated_time = mstime(); + long long key_count = dbTotalServerKeyCount(); + while (bytes_to_write) { + /* We write the AOF buffer in chunk of 8MB so that we can check the time in between them */ + size_t chunk_size = bytes_to_write < (8<<20) ? bytes_to_write : (8<<20); + + if (rioWrite(&aof,buf,chunk_size) == 0) + goto werr; + + bytes_to_write -= chunk_size; + buf += chunk_size; + + /* Update COW info */ + long long now = mstime(); + if (now - cow_updated_time >= 1000) { + sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, "AOF rewrite"); + cow_updated_time = now; + } + } /* Make sure data will not remain on the OS's output buffers */ if (fflush(fp)) goto werr; @@ -1623,6 +1752,7 @@ int rewriteAppendOnlyFile(char *filename) { serverLog(LL_NOTICE,"SYNC append only file rewrite performed"); stopSaving(1); return C_OK; +} // END GOTO SCOPED VARIABLES werr: serverLog(LL_WARNING,"Write error writing append only file on disk: %s", strerror(errno)); @@ -1739,7 +1869,6 @@ int rewriteAppendOnlyFileBackground(void) { if (hasActiveChildProcess()) return C_ERR; if (aofCreatePipes() != C_OK) return C_ERR; - openChildInfoPipe(); if ((childpid = redisFork(CHILD_TYPE_AOF)) == 0) { char tmpfile[256]; @@ -1748,7 +1877,7 @@ int rewriteAppendOnlyFileBackground(void) { redisSetCpuAffinity(g_pserver->aof_rewrite_cpulist); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid()); if (rewriteAppendOnlyFile(tmpfile) == C_OK) { - sendChildCOWInfo(CHILD_TYPE_AOF, "AOF rewrite"); + sendChildCowInfo(CHILD_INFO_TYPE_AOF_COW_SIZE, "AOF rewrite"); exitFromChild(0); } else { exitFromChild(1); @@ -1756,7 +1885,6 @@ int rewriteAppendOnlyFileBackground(void) { } else { /* Parent */ if (childpid == -1) { - closeChildInfoPipe(); serverLog(LL_WARNING, "Can't rewrite append only file in background: fork: %s", strerror(errno)); @@ -1764,10 +1892,9 @@ int rewriteAppendOnlyFileBackground(void) { return C_ERR; } serverLog(LL_NOTICE, - "Background append only file rewriting started by pid %d",childpid); + "Background append only file rewriting started by pid %ld",(long)childpid); g_pserver->aof_rewrite_scheduled = 0; g_pserver->aof_rewrite_time_start = time(NULL); - g_pserver->aof_child_pid = childpid; updateDictResizePolicy(); /* We set appendseldb to -1 in order to force the next call to the * feedAppendOnlyFile() to issue a SELECT command, so the differences @@ -1781,7 +1908,7 @@ int rewriteAppendOnlyFileBackground(void) { } void bgrewriteaofCommand(client *c) { - if (g_pserver->aof_child_pid != -1) { + if (g_pserver->child_type == CHILD_TYPE_AOF) { addReplyError(c,"Background append only file rewriting already in progress"); } else if (hasActiveChildProcess()) { g_pserver->aof_rewrite_scheduled = 1; @@ -1839,7 +1966,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { * rewritten AOF. */ latencyStartMonitor(latency); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", - (int)g_pserver->aof_child_pid); + (int)g_pserver->child_pid); newfd = open(tmpfile,O_WRONLY|O_APPEND); if (newfd == -1) { serverLog(LL_WARNING, @@ -1855,6 +1982,20 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { } latencyEndMonitor(latency); latencyAddSampleIfNeeded("aof-rewrite-diff-write",latency); + + if (g_pserver->aof_fsync == AOF_FSYNC_EVERYSEC) { + aof_background_fsync(newfd); + } else if (g_pserver->aof_fsync == AOF_FSYNC_ALWAYS) { + latencyStartMonitor(latency); + if (redis_fsync(newfd) == -1) { + serverLog(LL_WARNING, + "Error trying to fsync the parent diff to the rewritten AOF: %s", strerror(errno)); + close(newfd); + goto cleanup; + } + latencyEndMonitor(latency); + latencyAddSampleIfNeeded("aof-rewrite-done-fsync",latency); + } serverLog(LL_NOTICE, "Residual parent diff successfully flushed to the rewritten AOF (%.2f MB)", (double) aofRewriteBufferSize() / (1024*1024)); @@ -1922,14 +2063,11 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { /* AOF enabled, replace the old fd with the new one. */ oldfd = g_pserver->aof_fd; g_pserver->aof_fd = newfd; - if (g_pserver->aof_fsync == AOF_FSYNC_ALWAYS) - redis_fsync(newfd); - else if (g_pserver->aof_fsync == AOF_FSYNC_EVERYSEC) - aof_background_fsync(newfd); g_pserver->aof_selected_db = -1; /* Make sure SELECT is re-issued */ aofUpdateCurrentSize(); g_pserver->aof_rewrite_base_size = g_pserver->aof_current_size; g_pserver->aof_fsync_offset = g_pserver->aof_current_size; + g_pserver->aof_last_fsync = g_pserver->unixtime; /* Clear regular AOF buffer since its contents was just written to * the new AOF from the background rewrite buffer. */ @@ -1945,7 +2083,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { g_pserver->aof_state = AOF_ON; /* Asynchronously close the overwritten AOF. */ - if (oldfd != -1) bioCreateBackgroundJob(BIO_CLOSE_FILE,(void*)(long)oldfd,NULL,NULL); + if (oldfd != -1) bioCreateCloseJob(oldfd); serverLog(LL_VERBOSE, "Background AOF rewrite signal handler took %lldus", ustime()-now); @@ -1967,8 +2105,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { cleanup: aofClosePipes(); aofRewriteBufferReset(); - aofRemoveTempFile(g_pserver->aof_child_pid); - g_pserver->aof_child_pid = -1; + aofRemoveTempFile(g_pserver->child_pid); g_pserver->aof_rewrite_time_last = time(NULL)-g_pserver->aof_rewrite_time_start; g_pserver->aof_rewrite_time_start = -1; /* Schedule a new rewrite if we are waiting for it to switch the AOF ON. */ diff --git a/src/asciilogo.h b/src/asciilogo.h index 43ad70e79..7509cf518 100644 --- a/src/asciilogo.h +++ b/src/asciilogo.h @@ -41,7 +41,7 @@ const char *ascii_logo = " | / | \\ | https://docs.keydb.dev \n" " | / | \\ | \n" " | / | \\ | \n" -" (+)_ -- -- -- | -- -- -- _(+) \n" +" (+)_ -- -- -- | -- -- -- _(+) \n" " --_ | _-- \n" " --_ | _-- \n" " -(+)- %s\n" diff --git a/src/atomicvar.h b/src/atomicvar.h index ecd26ad70..222b82699 100644 --- a/src/atomicvar.h +++ b/src/atomicvar.h @@ -1,5 +1,5 @@ -/* This file implements atomic counters using __atomic or __sync macros if - * available, otherwise synchronizing different threads using a mutex. +/* This file implements atomic counters using c11 _Atomic, __atomic or __sync + * macros if available, otherwise we will throw an error when compile. * * The exported interface is composed of three macros: * @@ -8,16 +8,8 @@ * atomicDecr(var,count) -- Decrement the atomic counter * atomicGet(var,dstvar) -- Fetch the atomic counter value * atomicSet(var,value) -- Set the atomic counter value - * - * The variable 'var' should also have a declared mutex with the same - * name and the "_mutex" postfix, for instance: - * - * long myvar; - * pthread_mutex_t myvar_mutex; - * atomicSet(myvar,12345); - * - * If atomic primitives are available (tested in config.h) the mutex - * is not used. + * atomicGetWithSync(var,value) -- 'atomicGet' with inter-thread synchronization + * atomicSetWithSync(var,value) -- 'atomicSet' with inter-thread synchronization * * Never use return value from the macros, instead use the AtomicGetIncr() * if you need to get the current value and increment it atomically, like @@ -58,17 +50,64 @@ */ #include +#include "config.h" #ifndef __ATOMIC_VAR_H #define __ATOMIC_VAR_H +/* Define redisAtomic for atomic variable. */ +#define redisAtomic + /* To test Redis with Helgrind (a Valgrind tool) it is useful to define * the following macro, so that __sync macros are used: those can be detected * by Helgrind (even if they are less efficient) so that no false positive * is reported. */ // #define __ATOMIC_VAR_FORCE_SYNC_MACROS -#if !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && defined(__ATOMIC_RELAXED) && !defined(__sun) && (!defined(__clang__) || !defined(__APPLE__) || __apple_build_version__ > 4210057) +/* There will be many false positives if we test Redis with Helgrind, since + * Helgrind can't understand we have imposed ordering on the program, so + * we use macros in helgrind.h to tell Helgrind inter-thread happens-before + * relationship explicitly for avoiding false positives. + * + * For more details, please see: valgrind/helgrind.h and + * https://www.valgrind.org/docs/manual/hg-manual.html#hg-manual.effective-use + * + * These macros take effect only when 'make helgrind', and you must first + * install Valgrind in the default path configuration. */ +#ifdef __ATOMIC_VAR_FORCE_SYNC_MACROS +#include +#else +#define ANNOTATE_HAPPENS_BEFORE(v) ((void) v) +#define ANNOTATE_HAPPENS_AFTER(v) ((void) v) +#endif + +#if !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_ATOMICS__) +/* Use '_Atomic' keyword if the compiler supports. */ +#undef redisAtomic +#define redisAtomic _Atomic +/* Implementation using _Atomic in C11. */ + +#include +#define atomicIncr(var,count) atomic_fetch_add_explicit(&var,(count),memory_order_relaxed) +#define atomicGetIncr(var,oldvalue_var,count) do { \ + oldvalue_var = atomic_fetch_add_explicit(&var,(count),memory_order_relaxed); \ +} while(0) +#define atomicDecr(var,count) atomic_fetch_sub_explicit(&var,(count),memory_order_relaxed) +#define atomicGet(var,dstvar) do { \ + dstvar = atomic_load_explicit(&var,memory_order_relaxed); \ +} while(0) +#define atomicSet(var,value) atomic_store_explicit(&var,value,memory_order_relaxed) +#define atomicGetWithSync(var,dstvar) do { \ + dstvar = atomic_load_explicit(&var,memory_order_seq_cst); \ +} while(0) +#define atomicSetWithSync(var,value) \ + atomic_store_explicit(&var,value,memory_order_seq_cst) +#define REDIS_ATOMIC_API "c11-builtin" + +#elif !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && \ + (!defined(__clang__) || !defined(__APPLE__) || __apple_build_version__ > 4210057) && \ + defined(__ATOMIC_RELAXED) && defined(__ATOMIC_SEQ_CST) /* Implementation using __atomic macros. */ #define atomicIncr(var,count) __atomic_add_fetch(&var,(count),__ATOMIC_RELAXED) @@ -80,6 +119,11 @@ dstvar = __atomic_load_n(&var,__ATOMIC_RELAXED); \ } while(0) #define atomicSet(var,value) __atomic_store_n(&var,value,__ATOMIC_RELAXED) +#define atomicGetWithSync(var,dstvar) do { \ + dstvar = __atomic_load_n(&var,__ATOMIC_SEQ_CST); \ +} while(0) +#define atomicSetWithSync(var,value) \ + __atomic_store_n(&var,value,__ATOMIC_SEQ_CST) #define REDIS_ATOMIC_API "atomic-builtin" #elif defined(HAVE_ATOMIC) @@ -96,38 +140,19 @@ #define atomicSet(var,value) do { \ while(!__sync_bool_compare_and_swap(&var,var,value)); \ } while(0) +/* Actually the builtin issues a full memory barrier by default. */ +#define atomicGetWithSync(var,dstvar) { \ + dstvar = __sync_sub_and_fetch(&var,0,__sync_synchronize); \ + ANNOTATE_HAPPENS_AFTER(&var); \ +} while(0) +#define atomicSetWithSync(var,value) do { \ + ANNOTATE_HAPPENS_BEFORE(&var); \ + while(!__sync_bool_compare_and_swap(&var,var,value,__sync_synchronize)); \ +} while(0) #define REDIS_ATOMIC_API "sync-builtin" #else -/* Implementation using pthread mutex. */ - -#define atomicIncr(var,count) do { \ - pthread_mutex_lock(&var ## _mutex); \ - var += (count); \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define atomicGetIncr(var,oldvalue_var,count) do { \ - pthread_mutex_lock(&var ## _mutex); \ - oldvalue_var = var; \ - var += (count); \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define atomicDecr(var,count) do { \ - pthread_mutex_lock(&var ## _mutex); \ - var -= (count); \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define atomicGet(var,dstvar) do { \ - pthread_mutex_lock(&var ## _mutex); \ - dstvar = var; \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define atomicSet(var,value) do { \ - pthread_mutex_lock(&var ## _mutex); \ - var = value; \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define REDIS_ATOMIC_API "pthread-mutex" +#error "Unable to determine atomic operations for your platform" #endif #endif /* __ATOMIC_VAR_H */ diff --git a/src/bio.cpp b/src/bio.cpp index 2c28e57a0..17ca56d34 100644 --- a/src/bio.cpp +++ b/src/bio.cpp @@ -78,15 +78,13 @@ static unsigned long long bio_pending[BIO_NUM_OPS]; * file as the API does not expose the internals at all. */ struct bio_job { time_t time; /* Time at which the job was created. */ - /* Job specific arguments pointers. If we need to pass more than three - * arguments we can just pass a pointer to a structure or alike. */ - void *arg1, *arg2, *arg3; + /* Job specific arguments.*/ + int fd; /* Fd for file based background jobs */ + lazy_free_fn *free_fn; /* Function that will free the provided arguments */ + void *free_args[]; /* List of arguments to be passed to the free function */ }; void *bioProcessBackgroundJobs(void *arg); -void lazyfreeFreeObjectFromBioThread(robj *o); -void lazyfreeFreeDatabaseFromBioThread(dict *ht1, expireset *set); -void lazyfreeFreeSlotsMapFromBioThread(rax *rt); /* Make sure we have enough stack to perform all the things we do in the * main thread. */ @@ -128,13 +126,8 @@ void bioInit(void) { } } -void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3) { - struct bio_job *job = (bio_job*)zmalloc(sizeof(*job), MALLOC_LOCAL); - +void bioSubmitJob(int type, struct bio_job *job) { job->time = time(NULL); - job->arg1 = arg1; - job->arg2 = arg2; - job->arg3 = arg3; pthread_mutex_lock(&bio_mutex[type]); listAddNodeTail(bio_jobs[type],job); bio_pending[type]++; @@ -142,6 +135,35 @@ void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3) { pthread_mutex_unlock(&bio_mutex[type]); } +void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...) { + va_list valist; + /* Allocate memory for the job structure and all required + * arguments */ + struct bio_job *job = (bio_job*)zmalloc(sizeof(*job) + sizeof(void *) * (arg_count)); + job->free_fn = free_fn; + + va_start(valist, arg_count); + for (int i = 0; i < arg_count; i++) { + job->free_args[i] = va_arg(valist, void *); + } + va_end(valist); + bioSubmitJob(BIO_LAZY_FREE, job); +} + +void bioCreateCloseJob(int fd) { + struct bio_job *job = (bio_job*)zmalloc(sizeof(*job)); + job->fd = fd; + + bioSubmitJob(BIO_CLOSE_FILE, job); +} + +void bioCreateFsyncJob(int fd) { + struct bio_job *job = (bio_job*)zmalloc(sizeof(*job)); + job->fd = fd; + + bioSubmitJob(BIO_AOF_FSYNC, job); +} + void *bioProcessBackgroundJobs(void *arg) { struct bio_job *job; unsigned long type = (unsigned long) arg; @@ -196,20 +218,11 @@ void *bioProcessBackgroundJobs(void *arg) { /* Process the job accordingly to its type. */ if (type == BIO_CLOSE_FILE) { - close((long)job->arg1); + close(job->fd); } else if (type == BIO_AOF_FSYNC) { - redis_fsync((long)job->arg1); + redis_fsync(job->fd); } else if (type == BIO_LAZY_FREE) { - /* What we free changes depending on what arguments are set: - * arg1 -> free the object at pointer. - * arg2 & arg3 -> free two dictionaries (a Redis DB). - * only arg3 -> free the radix tree. */ - if (job->arg1) - lazyfreeFreeObjectFromBioThread((robj*)job->arg1); - else if (job->arg2 && job->arg3) - lazyfreeFreeDatabaseFromBioThread((dict*)job->arg2,(expireset*)job->arg3); - else if (job->arg3) - lazyfreeFreeSlotsMapFromBioThread((rax*)job->arg3); + job->free_fn(job->free_args); } else { serverPanic("Wrong job type in bioProcessBackgroundJobs()."); } diff --git a/src/bio.h b/src/bio.h index bf350f9de..b0ce9bbad 100644 --- a/src/bio.h +++ b/src/bio.h @@ -32,13 +32,17 @@ extern "C" { #endif +typedef void lazy_free_fn(void *args[]); + /* Exported API */ void bioInit(void); -void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3); unsigned long long bioPendingJobsOfType(int type); unsigned long long bioWaitStepOfType(int type); time_t bioOlderJobOfType(int type); void bioKillThreads(void); +void bioCreateCloseJob(int fd); +void bioCreateFsyncJob(int fd); +void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...); /* Background job opcodes */ #define BIO_CLOSE_FILE 0 /* Deferred close(2) syscall. */ diff --git a/src/bitops.cpp b/src/bitops.cpp index 8b26114db..c03f068fd 100644 --- a/src/bitops.cpp +++ b/src/bitops.cpp @@ -36,7 +36,7 @@ /* Count number of bits set in the binary array pointed by 's' and long * 'count' bytes. The implementation of this function is required to - * work with an input string length up to 512 MB. */ + * work with an input string length up to 512 MB or more (server.proto_max_bulk_len) */ size_t redisPopcount(const void *s, long count) { size_t bits = 0; unsigned char *p = (unsigned char*)s; @@ -407,7 +407,7 @@ void printBits(unsigned char *p, unsigned long count) { /* This helper function used by GETBIT / SETBIT parses the bit offset argument * making sure an error is returned if it is negative or if it overflows - * Redis 512 MB limit for the string value. + * Redis 512 MB limit for the string value or more (server.proto_max_bulk_len). * * If the 'hash' argument is true, and 'bits is positive, then the command * will also parse bit offsets prefixed by "#". In such a case the offset @@ -430,8 +430,8 @@ int getBitOffsetFromArgument(client *c, robj *o, size_t *offset, int hash, int b /* Adjust the offset by 'bits' for # form. */ if (usehash) loffset *= bits; - /* Limit offset to 512MB in bytes */ - if ((loffset < 0) || ((unsigned long long)loffset >> 3) >= (512*1024*1024)) + /* Limit offset to server.proto_max_bulk_len (512MB in bytes by default) */ + if ((loffset < 0) || (loffset >> 3) >= g_pserver->proto_max_bulk_len) { addReplyError(c,err); return C_ERR; @@ -482,12 +482,12 @@ int getBitfieldTypeFromArgument(client *c, robj *o, int *sign, int *bits) { robj *lookupStringForBitCommand(client *c, size_t maxbit) { size_t byte = maxbit >> 3; robj *o = lookupKeyWrite(c->db,c->argv[1]); + if (checkType(c,o,OBJ_STRING)) return NULL; if (o == NULL) { o = createObject(OBJ_STRING,sdsnewlen(NULL, byte+1)); dbAdd(c->db,c->argv[1],o); } else { - if (checkType(c,o,OBJ_STRING)) return NULL; o = dbUnshareStringValue(c->db,c->argv[1],o); o->m_ptr = sdsgrowzero(szFromObj(o),byte+1); } @@ -619,7 +619,7 @@ void bitopCommand(client *c) { else if (!strcasecmp(opname, "rshift")) op = BITOP_RSHIFT; else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } @@ -731,7 +731,6 @@ void bitopCommand(client *c) { /* Compute the bit operation, if at least one string is not empty. */ if (maxlen) { res = (unsigned char*) sdsnewlen(NULL,maxlen); - unsigned char output, byte; unsigned long i; /* Fast path: as far as we have data for all the input bitmaps we @@ -801,22 +800,33 @@ void bitopCommand(client *c) { } } } - #endif + } + #endif - /* j is set to the next byte to process by the previous loop. */ - for (; j < maxlen; j++) { - output = (len[0] <= j) ? 0 : src[0][j]; - if (op == BITOP_NOT) output = ~output; - for (i = 1; i < numkeys; i++) { - byte = (len[i] <= j) ? 0 : src[i][j]; - switch(op) { - case BITOP_AND: output &= byte; break; - case BITOP_OR: output |= byte; break; - case BITOP_XOR: output ^= byte; break; - } + /* j is set to the next byte to process by the previous loop. */ + for (; j < maxlen; j++) { + auto output = (len[0] <= j) ? 0 : src[0][j]; + if (op == BITOP_NOT) output = ~output; + for (unsigned long i = 1; i < numkeys; i++) { + int skip = 0; + auto byte = (len[i] <= j) ? 0 : src[i][j]; + switch(op) { + case BITOP_AND: + output &= byte; + skip = (output == 0); + break; + case BITOP_OR: + output |= byte; + skip = (output == 0xff); + break; + case BITOP_XOR: output ^= byte; break; + } + + if (skip) { + break; } - res[j] = output; } + res[j] = output; } } for (j = 0; j < numkeys; j++) { @@ -876,7 +886,7 @@ void bitcountCommand(client *c) { end = strlen-1; } else { /* Syntax error. */ - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } @@ -941,7 +951,7 @@ void bitposCommand(client *c) { end = strlen-1; } else { /* Syntax error. */ - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } @@ -1033,7 +1043,7 @@ void bitfieldGeneric(client *c, int flags) { } continue; } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); zfree(ops); return; } diff --git a/src/blocked.cpp b/src/blocked.cpp index ecde109dc..c4ca8492c 100644 --- a/src/blocked.cpp +++ b/src/blocked.cpp @@ -61,9 +61,13 @@ */ #include "server.h" +#include "slowlog.h" +#include "latency.h" +#include "monotonic.h" #include -int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int where); +int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int wherefrom, int whereto); +int getListPositionFromObjectOrReply(client *c, robj *arg, int *position); /* This structure represents the blocked key information that we store * in the client structure. Each client blocked on keys, has a @@ -90,6 +94,26 @@ void blockClient(client *c, int btype) { g_pserver->blocked_clients++; g_pserver->blocked_clients_by_type[btype]++; addClientToTimeoutTable(c); + if (btype == BLOCKED_PAUSE) { + listAddNodeTail(g_pserver->paused_clients, c); + c->paused_list_node = listLast(g_pserver->paused_clients); + /* Mark this client to execute its command */ + c->flags |= CLIENT_PENDING_COMMAND; + } +} + +/* This function is called after a client has finished a blocking operation + * in order to update the total command duration, log the command into + * the Slow log if needed, and log the reply duration event if needed. */ +void updateStatsOnUnblock(client *c, long blocked_us, long reply_us){ + const ustime_t total_cmd_duration = c->duration + blocked_us + reply_us; + c->lastcmd->microseconds += total_cmd_duration; + /* Log the command into the Slow log if needed. */ + if (!(c->lastcmd->flags & CMD_SKIP_SLOWLOG)) { + slowlogPushEntryIfNeeded(c,c->argv,c->argc,total_cmd_duration); + /* Log the reply duration event. */ + latencyAddSampleIfNeeded("command-unblocking",reply_us/1000); + } } /* This function is called in the beforeSleep() function of the event loop @@ -118,6 +142,11 @@ void processUnblockedClients(int iel) { * client is not blocked before to proceed, but things may change and * the code is conceptually more correct this way. */ if (!(c->flags & CLIENT_BLOCKED)) { + /* If we have a queued command, execute it now. */ + if (processPendingCommandsAndResetClient(c, CMD_CALL_FULL) == C_ERR) { + continue; + } + /* Then process client if it has more data in it's buffer. */ if (c->querybuf && sdslen(c->querybuf) > 0) { processInputBuffer(c, CMD_CALL_FULL); } @@ -168,6 +197,9 @@ void unblockClient(client *c) { } else if (c->btype == BLOCKED_MODULE) { if (moduleClientIsBlockedOnKeys(c)) unblockClientWaitingData(c); unblockClientFromModule(c); + } else if (c->btype == BLOCKED_PAUSE) { + listDelNode(g_pserver->paused_clients,c->paused_list_node); + c->paused_list_node = NULL; } else { serverPanic("Unknown btype in unblockClient()."); } @@ -216,9 +248,16 @@ void disconnectAllBlockedClients(void) { fastlock_lock(&c->lock); if (c->flags & CLIENT_BLOCKED) { - addReplySds(c,sdsnew( + /* PAUSED clients are an exception, when they'll be unblocked, the + * command processing will start from scratch, and the command will + * be either executed or rejected. (unlike LIST blocked clients for + * which the command is already in progress in a way. */ + if (c->btype == BLOCKED_PAUSE) + continue; + + addReplyError(c, "-UNBLOCKED force unblock from blocking operation, " - "instance state changed (master -> replica?)\r\n")); + "instance state changed (master -> replica?)"); unblockClient(c); c->flags |= CLIENT_CLOSE_AFTER_REPLY; } @@ -250,10 +289,9 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) { } robj *dstkey = receiver->bpop.target; - int where = (receiver->lastcmd && - receiver->lastcmd->proc == blpopCommand) ? - LIST_HEAD : LIST_TAIL; - robj *value = listTypePop(o,where); + int wherefrom = receiver->bpop.listpos.wherefrom; + int whereto = receiver->bpop.listpos.whereto; + robj *value = listTypePop(o, wherefrom); if (value) { /* Protect receiver->bpop.target, that will be @@ -262,14 +300,17 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) { if (dstkey) incrRefCount(dstkey); unblockClient(receiver); + monotime replyTimer; + elapsedStart(&replyTimer); if (serveClientBlockedOnList(receiver, rl->key,dstkey,rl->db,value, - where) == C_ERR) + wherefrom, whereto) == C_ERR) { /* If we failed serving the client we need * to also undo the POP operation. */ - listTypePush(o,value,where); + listTypePush(o,value,wherefrom); } + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); if (dstkey) decrRefCount(dstkey); decrRefCount(value); @@ -315,7 +356,10 @@ void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { receiver->lastcmd->proc == bzpopminCommand) ? ZSET_MIN : ZSET_MAX; unblockClient(receiver); + monotime replyTimer; + elapsedStart(&replyTimer); genericZpopCommand(receiver,&rl->key,1,where,1,NULL); + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); zcard--; /* Replicate the command. */ @@ -392,13 +436,22 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { int noack = 0; if (group) { + int created = 0; consumer = streamLookupConsumer(group, szFromObj(receiver->bpop.xread_consumer), - SLC_NONE); + SLC_NONE, + &created); noack = receiver->bpop.xread_group_noack; + if (created && noack) { + streamPropagateConsumerCreation(receiver,rl->key, + receiver->bpop.xread_group, + consumer->name); + } } + monotime replyTimer; + elapsedStart(&replyTimer); /* Emit the two elements sub-array consisting of * the name of the stream and the data we * extracted from it. Wrapped in a single-item @@ -418,6 +471,7 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { streamReplyWithRange(receiver,s,&start,NULL, receiver->bpop.xread_count, 0, group, consumer, noack, &pi); + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); /* Note that after we unblock the client, 'gt' * and other receiver->bpop stuff are no longer @@ -464,7 +518,10 @@ void serveClientsBlockedOnKeyByModule(readyList *rl) { * different modules with different triggers to consider if a key * is ready or not. This means we can't exit the loop but need * to continue after the first failure. */ + monotime replyTimer; + elapsedStart(&replyTimer); if (!moduleTryServeClientBlockedOnKey(receiver, rl->key)) continue; + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); moduleUnblockClient(receiver); } @@ -480,8 +537,8 @@ void serveClientsBlockedOnKeyByModule(readyList *rl) { * one new element via some write operation are accumulated into * the g_pserver->ready_keys list. This function will run the list and will * serve clients accordingly. Note that the function will iterate again and - * again as a result of serving BRPOPLPUSH we can have new blocking clients - * to serve because of the PUSH side of BRPOPLPUSH. + * again as a result of serving BLMOVE we can have new blocking clients + * to serve because of the PUSH side of BLMOVE. * * This function is normally "fair", that is, it will server clients * using a FIFO behavior. However this fairness is violated in certain @@ -515,7 +572,7 @@ void handleClientsBlockedOnKeys(void) { /* Even if we are not inside call(), increment the call depth * in order to make sure that keys are expired against a fixed * reference time, and not against the wallclock time. This - * way we can lookup an object multiple times (BRPOPLPUSH does + * way we can lookup an object multiple times (BLMOVE does * that) without the risk of it being freed in the second * lookup, invalidating the first one. * See https://github.com/antirez/redis/pull/6554. */ @@ -575,7 +632,7 @@ void handleClientsBlockedOnKeys(void) { * stream keys, we also provide an array of streamID structures: clients will * be unblocked only when items with an ID greater or equal to the specified * one is appended to the stream. */ -void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids) { +void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, struct listPos *listpos, streamID *ids) { dictEntry *de; list *l; int j; @@ -583,6 +640,8 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo c->bpop.timeout = timeout; c->bpop.target = target; + if (listpos != NULL) c->bpop.listpos = *listpos; + if (target != NULL) incrRefCount(target); for (j = 0; j < numkeys; j++) { @@ -656,6 +715,16 @@ void unblockClientWaitingData(client *c) { } } +static int getBlockedTypeByType(int type) { + switch (type) { + case OBJ_LIST: return BLOCKED_LIST; + case OBJ_ZSET: return BLOCKED_ZSET; + case OBJ_MODULE: return BLOCKED_MODULE; + case OBJ_STREAM: return BLOCKED_STREAM; + default: return BLOCKED_NONE; + } +} + /* If the specified key has clients blocked waiting for list pushes, this * function will put the key reference into the g_pserver->ready_keys list. * Note that db->ready_keys is a hash table that allows us to avoid putting @@ -663,9 +732,24 @@ void unblockClientWaitingData(client *c) { * made by a script or in the context of MULTI/EXEC. * * The list will be finally processed by handleClientsBlockedOnKeys() */ -void signalKeyAsReady(redisDb *db, robj *key) { +void signalKeyAsReady(redisDb *db, robj *key, int type) { readyList *rl; + /* Quick returns. */ + int btype = getBlockedTypeByType(type); + if (btype == BLOCKED_NONE) { + /* The type can never block. */ + return; + } + if (!g_pserver->blocked_clients_by_type[btype] && + !g_pserver->blocked_clients_by_type[BLOCKED_MODULE]) { + /* No clients block on this type. Note: Blocked modules are represented + * by BLOCKED_MODULE, even if the intention is to wake up by normal + * types (list, zset, stream), so we need to check that there are no + * blocked modules before we do a quick return here. */ + return; + } + /* No clients blocking for this key? No need to queue it. */ if (dictFind(db->blocking_keys,key) == NULL) return; @@ -693,4 +777,8 @@ void signalKeyAsReady(redisDb *db, robj *key) { serverAssert(dictAdd(db->ready_keys,key,NULL) == DICT_OK); } - +void signalKeyAsReady(redisDb *db, sds key, int type) { + redisObjectStack o; + initStaticStringObject(o, key); + signalKeyAsReady(db, &o, type); +} \ No newline at end of file diff --git a/src/childinfo.cpp b/src/childinfo.cpp index 77900ac11..9e81d395a 100644 --- a/src/childinfo.cpp +++ b/src/childinfo.cpp @@ -30,18 +30,25 @@ #include "server.h" #include +typedef struct { + size_t keys; + size_t cow; + double progress; + childInfoType information_type; /* Type of information */ +} child_info_data; + /* Open a child-parent channel used in order to move information about the * RDB / AOF saving process from the child to the parent (for instance * the amount of copy on write memory used) */ void openChildInfoPipe(void) { if (pipe(g_pserver->child_info_pipe) == -1) { /* On error our two file descriptors should be still set to -1, - * but we call anyway cloesChildInfoPipe() since can't hurt. */ + * but we call anyway closeChildInfoPipe() since can't hurt. */ closeChildInfoPipe(); } else if (anetNonBlock(NULL,g_pserver->child_info_pipe[0]) != ANET_OK) { closeChildInfoPipe(); } else { - memset(&g_pserver->child_info_data,0,sizeof(g_pserver->child_info_data)); + g_pserver->child_info_nread = 0; } } @@ -54,34 +61,88 @@ void closeChildInfoPipe(void) { close(g_pserver->child_info_pipe[1]); g_pserver->child_info_pipe[0] = -1; g_pserver->child_info_pipe[1] = -1; + g_pserver->child_info_nread = 0; } } -/* Send COW data to parent. The child should call this function after populating - * the corresponding fields it want to sent (according to the process type). */ -void sendChildInfo(int ptype) { +/* Send save data to parent. */ +void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, const char *pname) { if (g_pserver->child_info_pipe[1] == -1) return; - g_pserver->child_info_data.magic = CHILD_INFO_MAGIC; - g_pserver->child_info_data.process_type = ptype; - ssize_t wlen = sizeof(g_pserver->child_info_data); - if (write(g_pserver->child_info_pipe[1],&g_pserver->child_info_data,wlen) != wlen) { + + child_info_data data = {0}; /* zero everything, including padding to sattisfy valgrind */ + data.information_type = info_type; + data.keys = keys; + data.cow = zmalloc_get_private_dirty(-1); + data.progress = progress; + + if (data.cow) { + serverLog((info_type == CHILD_INFO_TYPE_CURRENT_INFO) ? LL_VERBOSE : LL_NOTICE, + "%s: %zu MB of memory used by copy-on-write", + pname, data.cow/(1024*1024)); + } + + ssize_t wlen = sizeof(data); + + if (write(g_pserver->child_info_pipe[1], &data, wlen) != wlen) { /* Nothing to do on error, this will be detected by the other side. */ } } -/* Receive COW data from parent. */ -void receiveChildInfo(void) { - if (g_pserver->child_info_pipe[0] == -1) return; - ssize_t wlen = sizeof(g_pserver->child_info_data); - if (read(g_pserver->child_info_pipe[0],&g_pserver->child_info_data,wlen) == wlen && - g_pserver->child_info_data.magic == CHILD_INFO_MAGIC) - { - if (g_pserver->child_info_data.process_type == CHILD_TYPE_RDB) { - g_pserver->stat_rdb_cow_bytes = g_pserver->child_info_data.cow_size; - } else if (g_pserver->child_info_data.process_type == CHILD_TYPE_AOF) { - g_pserver->stat_aof_cow_bytes = g_pserver->child_info_data.cow_size; - } else if (g_pserver->child_info_data.process_type == CHILD_TYPE_MODULE) { - g_pserver->stat_module_cow_bytes = g_pserver->child_info_data.cow_size; - } +/* Update Child info. */ +void updateChildInfo(childInfoType information_type, size_t cow, size_t keys, double progress) { + if (information_type == CHILD_INFO_TYPE_CURRENT_INFO) { + g_pserver->stat_current_cow_bytes = cow; + g_pserver->stat_current_save_keys_processed = keys; + if (progress != -1) g_pserver->stat_module_progress = progress; + } else if (information_type == CHILD_INFO_TYPE_AOF_COW_SIZE) { + g_pserver->stat_aof_cow_bytes = cow; + } else if (information_type == CHILD_INFO_TYPE_RDB_COW_SIZE) { + g_pserver->stat_rdb_cow_bytes = cow; + } else if (information_type == CHILD_INFO_TYPE_MODULE_COW_SIZE) { + g_pserver->stat_module_cow_bytes = cow; + } +} + +/* Read child info data from the pipe. + * if complete data read into the buffer, + * data is stored into *buffer, and returns 1. + * otherwise, the partial data is left in the buffer, waiting for the next read, and returns 0. */ +int readChildInfo(childInfoType *information_type, size_t *cow, size_t *keys, double* progress) { + /* We are using here a static buffer in combination with the server.child_info_nread to handle short reads */ + static child_info_data buffer; + ssize_t wlen = sizeof(buffer); + + /* Do not overlap */ + if (g_pserver->child_info_nread == wlen) g_pserver->child_info_nread = 0; + + int nread = read(g_pserver->child_info_pipe[0], (char *)&buffer + g_pserver->child_info_nread, wlen - g_pserver->child_info_nread); + if (nread > 0) { + g_pserver->child_info_nread += nread; + } + + /* We have complete child info */ + if (g_pserver->child_info_nread == wlen) { + *information_type = buffer.information_type; + *cow = buffer.cow; + *keys = buffer.keys; + *progress = buffer.progress; + return 1; + } else { + return 0; + } +} + +/* Receive info data from child. */ +void receiveChildInfo(void) { + if (g_pserver->child_info_pipe[0] == -1) return; + + size_t cow; + size_t keys; + double progress; + childInfoType information_type; + + /* Drain the pipe and update child info so that we get the final message. */ + while (readChildInfo(&information_type, &cow, &keys, &progress)) { + updateChildInfo(information_type, cow, keys, progress); } } diff --git a/src/cli_common.c b/src/cli_common.c new file mode 100644 index 000000000..e88327ace --- /dev/null +++ b/src/cli_common.c @@ -0,0 +1,194 @@ +/* CLI (command line interface) common methods + * + * Copyright (c) 2020, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "cli_common.h" +#include +#include +#include /* Use hiredis' sds compat header that maps sds calls to their hi_ variants */ +#include /* use sds.h from hiredis, so that only one set of sds functions will be present in the binary */ +#ifdef USE_OPENSSL +#include +#include +#include +#endif + + +/* Wrapper around redisSecureConnection to avoid hiredis_ssl dependencies if + * not building with TLS support. + */ +int cliSecureConnection(redisContext *c, cliSSLconfig config, const char **err) { +#ifdef USE_OPENSSL + static SSL_CTX *ssl_ctx = NULL; + + if (!ssl_ctx) { + ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + if (!ssl_ctx) { + *err = "Failed to create SSL_CTX"; + goto error; + } + SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); + SSL_CTX_set_verify(ssl_ctx, config.skip_cert_verify ? SSL_VERIFY_NONE : SSL_VERIFY_PEER, NULL); + + if (config.cacert || config.cacertdir) { + if (!SSL_CTX_load_verify_locations(ssl_ctx, config.cacert, config.cacertdir)) { + *err = "Invalid CA Certificate File/Directory"; + goto error; + } + } else { + if (!SSL_CTX_set_default_verify_paths(ssl_ctx)) { + *err = "Failed to use default CA paths"; + goto error; + } + } + + if (config.cert && !SSL_CTX_use_certificate_chain_file(ssl_ctx, config.cert)) { + *err = "Invalid client certificate"; + goto error; + } + + if (config.key && !SSL_CTX_use_PrivateKey_file(ssl_ctx, config.key, SSL_FILETYPE_PEM)) { + *err = "Invalid private key"; + goto error; + } + if (config.ciphers && !SSL_CTX_set_cipher_list(ssl_ctx, config.ciphers)) { + *err = "Error while configuring ciphers"; + goto error; + } +#ifdef TLS1_3_VERSION + if (config.ciphersuites && !SSL_CTX_set_ciphersuites(ssl_ctx, config.ciphersuites)) { + *err = "Error while setting cypher suites"; + goto error; + } +#endif + } + + SSL *ssl = SSL_new(ssl_ctx); + if (!ssl) { + *err = "Failed to create SSL object"; + return REDIS_ERR; + } + + if (config.sni && !SSL_set_tlsext_host_name(ssl, config.sni)) { + *err = "Failed to configure SNI"; + SSL_free(ssl); + return REDIS_ERR; + } + + return redisInitiateSSL(c, ssl); + +error: + SSL_CTX_free(ssl_ctx); + ssl_ctx = NULL; + return REDIS_ERR; +#else + (void) config; + (void) c; + (void) err; + return REDIS_OK; +#endif +} + +/* Wrapper around hiredis to allow arbitrary reads and writes. + * + * We piggybacks on top of hiredis to achieve transparent TLS support, + * and use its internal buffers so it can co-exist with commands + * previously/later issued on the connection. + * + * Interface is close to enough to read()/write() so things should mostly + * work transparently. + */ + +/* Write a raw buffer through a redisContext. If we already have something + * in the buffer (leftovers from hiredis operations) it will be written + * as well. + */ +ssize_t cliWriteConn(redisContext *c, const char *buf, size_t buf_len) +{ + int done = 0; + + /* Append data to buffer which is *usually* expected to be empty + * but we don't assume that, and write. + */ + c->obuf = sdscatlen(c->obuf, buf, buf_len); + if (redisBufferWrite(c, &done) == REDIS_ERR) { + if (!(c->flags & REDIS_BLOCK)) + errno = EAGAIN; + + /* On error, we assume nothing was written and we roll back the + * buffer to its original state. + */ + if (sdslen(c->obuf) > buf_len) + sdsrange(c->obuf, 0, -(buf_len+1)); + else + sdsclear(c->obuf); + + return -1; + } + + /* If we're done, free up everything. We may have written more than + * buf_len (if c->obuf was not initially empty) but we don't have to + * tell. + */ + if (done) { + sdsclear(c->obuf); + return buf_len; + } + + /* Write was successful but we have some leftovers which we should + * remove from the buffer. + * + * Do we still have data that was there prior to our buf? If so, + * restore buffer to it's original state and report no new data was + * writen. + */ + if (sdslen(c->obuf) > buf_len) { + sdsrange(c->obuf, 0, -(buf_len+1)); + return 0; + } + + /* At this point we're sure no prior data is left. We flush the buffer + * and report how much we've written. + */ + size_t left = sdslen(c->obuf); + sdsclear(c->obuf); + return buf_len - left; +} + +/* Wrapper around OpenSSL (libssl and libcrypto) initialisation + */ +int cliSecureInit() +{ +#ifdef USE_OPENSSL + ERR_load_crypto_strings(); + SSL_load_error_strings(); + SSL_library_init(); +#endif + return REDIS_OK; +} diff --git a/src/cli_common.h b/src/cli_common.h new file mode 100644 index 000000000..77f28d805 --- /dev/null +++ b/src/cli_common.h @@ -0,0 +1,58 @@ +#ifndef __CLICOMMON_H +#define __CLICOMMON_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct cliSSLconfig { + /* Requested SNI, or NULL */ + char *sni; + /* CA Certificate file, or NULL */ + char *cacert; + /* Directory where trusted CA certificates are stored, or NULL */ + char *cacertdir; + /* Skip server certificate verification. */ + int skip_cert_verify; + /* Client certificate to authenticate with, or NULL */ + char *cert; + /* Private key file to authenticate with, or NULL */ + char *key; + /* Prefered cipher list, or NULL (applies only to <= TLSv1.2) */ + char* ciphers; + /* Prefered ciphersuites list, or NULL (applies only to TLSv1.3) */ + char* ciphersuites; +} cliSSLconfig; + +/* Wrapper around redisSecureConnection to avoid hiredis_ssl dependencies if + * not building with TLS support. + */ +int cliSecureConnection(redisContext *c, cliSSLconfig config, const char **err); + +/* Wrapper around hiredis to allow arbitrary reads and writes. + * + * We piggybacks on top of hiredis to achieve transparent TLS support, + * and use its internal buffers so it can co-exist with commands + * previously/later issued on the connection. + * + * Interface is close to enough to read()/write() so things should mostly + * work transparently. + */ + +/* Write a raw buffer through a redisContext. If we already have something + * in the buffer (leftovers from hiredis operations) it will be written + * as well. + */ +ssize_t cliWriteConn(redisContext *c, const char *buf, size_t buf_len); + +/* Wrapper around OpenSSL (libssl and libcrypto) initialisation. + */ +int cliSecureInit(); + +#ifdef __cplusplus +} +#endif + +#endif /* __CLICOMMON_H */ diff --git a/src/cluster.cpp b/src/cluster.cpp index f51a62d42..72867d9d1 100644 --- a/src/cluster.cpp +++ b/src/cluster.cpp @@ -47,7 +47,7 @@ clusterNode *myself = NULL; clusterNode *createClusterNode(char *nodename, int flags); -int clusterAddNode(clusterNode *node); +void clusterAddNode(clusterNode *node); void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask); void clusterReadHandler(connection *conn); void clusterSendPing(clusterLink *link, int type); @@ -415,7 +415,7 @@ int clusterLockConfig(char *filename) { /* To lock it, we need to open the file in a way it is created if * it does not exist, otherwise there is a race condition with other * processes. */ - int fd = open(filename,O_WRONLY|O_CREAT,0644); + int fd = open(filename,O_WRONLY|O_CREAT|O_CLOEXEC,0644); if (fd == -1) { serverLog(LL_WARNING, "Can't open %s in order to acquire a lock: %s", @@ -519,7 +519,7 @@ void clusterInit(void) { if (saveconf) clusterSaveConfigOrDie(1); /* We need a listening TCP port for our cluster messaging needs. */ - g_pserver->cfd_count = 0; + g_pserver->cfd.count = 0; /* Port sanity check II * The other handshake port check is triggered too late to stop @@ -529,23 +529,16 @@ void clusterInit(void) { serverLog(LL_WARNING, "KeyDB port number too high. " "Cluster communication port is 10,000 port " "numbers higher than your KeyDB port. " - "Your KeyDB port number must be " - "lower than 55535."); + "Your KeyDB port number must be 55535 or less."); exit(1); } - if (listenToPort(port+CLUSTER_PORT_INCR, - g_pserver->cfd,&g_pserver->cfd_count, 0 /*fReusePort*/, 0 /*fFirstListen*/) == C_ERR) - { + if (listenToPort(port+CLUSTER_PORT_INCR, &g_pserver->cfd, 0 /*fReusePort*/, 0 /*fFirstListen*/) == C_ERR) { exit(1); - } else { - int j; - - for (j = 0; j < g_pserver->cfd_count; j++) { - if (aeCreateFileEvent(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, g_pserver->cfd[j], AE_READABLE, - clusterAcceptHandler, NULL) == AE_ERR) - serverPanic("Unrecoverable error creating Redis Cluster " - "file event."); - } + } + + serverAssert(serverTL == &g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN]); + if (createSocketAcceptHandler(&g_pserver->cfd, clusterAcceptHandler) != C_OK) { + serverPanic("Unrecoverable error creating Redis Cluster socket accept handler."); } /* The slots -> keys map is a radix tree. Initialize it here. */ @@ -815,6 +808,7 @@ clusterNode *createClusterNode(char *nodename, int flags) { node->configEpoch = 0; node->flags = flags; memset(node->slots,0,sizeof(node->slots)); + node->slots_info = NULL; node->numslots = 0; node->numslaves = 0; node->slaves = NULL; @@ -997,12 +991,12 @@ void freeClusterNode(clusterNode *n) { } /* Add a node to the nodes hash table */ -int clusterAddNode(clusterNode *node) { +void clusterAddNode(clusterNode *node) { int retval; retval = dictAdd(g_pserver->cluster->nodes, sdsnewlen(node->name,CLUSTER_NAMELEN), node); - return (retval == DICT_OK) ? C_OK : C_ERR; + serverAssert(retval == DICT_OK); } /* Remove a node from the cluster. The function performs the high level @@ -1862,6 +1856,7 @@ int clusterProcessPacket(clusterLink *link) { g_pserver->cluster->mf_master_offset == 0) { g_pserver->cluster->mf_master_offset = sender->repl_offset; + clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_MANUALFAILOVER); serverLog(LL_WARNING, "Received replication offset for paused " "master manual failover: %lld", @@ -2165,7 +2160,7 @@ int clusterProcessPacket(clusterLink *link) { /* Don't bother creating useless objects if there are no * Pub/Sub subscribers. */ if (dictSize(g_pserver->pubsub_channels) || - listLength(g_pserver->pubsub_patterns)) + dictSize(g_pserver->pubsub_patterns)) { channel_len = ntohl(hdr->data.publish.msg.channel_len); message_len = ntohl(hdr->data.publish.msg.message_len); @@ -2203,9 +2198,15 @@ int clusterProcessPacket(clusterLink *link) { resetManualFailover(); g_pserver->cluster->mf_end = now + CLUSTER_MF_TIMEOUT; g_pserver->cluster->mf_slave = sender; - pauseClients(now+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT)); + pauseClients(now+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT),CLIENT_PAUSE_WRITE); serverLog(LL_WARNING,"Manual failover requested by replica %.40s.", sender->name); + /* We need to send a ping message to the replica, as it would carry + * `server.cluster->mf_master_offset`, which means the master paused clients + * at offset `server.cluster->mf_master_offset`, so that the replica would + * know that it is safe to set its `server.cluster->mf_can_start` to 1 so as + * to complete failover as quickly as possible. */ + clusterSendPing(link, CLUSTERMSG_TYPE_PING); } else if (type == CLUSTERMSG_TYPE_UPDATE) { clusterNode *n; /* The node the update is about. */ uint64_t reportedConfigEpoch = @@ -2856,7 +2857,7 @@ void clusterPropagatePublish(robj *channel, robj *message) { * SLAVE node specific functions * -------------------------------------------------------------------------- */ -/* This function sends a FAILOVE_AUTH_REQUEST message to every node in order to +/* This function sends a FAILOVER_AUTH_REQUEST message to every node in order to * see if there is the quorum for this slave instance to failover its failing * master. * @@ -3462,9 +3463,8 @@ void clusterHandleSlaveMigration(int max_slaves) { * The function can be used both to initialize the manual failover state at * startup or to abort a manual failover in progress. */ void resetManualFailover(void) { - if (g_pserver->cluster->mf_end && clientsArePaused()) { - g_pserver->clients_pause_end_time = 0; - unpauseClientsIfNecessary(); + if (g_pserver->cluster->mf_end) { + checkClientPauseTimeoutAndReturnIfPaused(); } g_pserver->cluster->mf_end = 0; /* No manual failover in progress. */ g_pserver->cluster->mf_can_start = 0; @@ -3499,7 +3499,10 @@ void clusterHandleManualFailover(void) { serverLog(LL_WARNING, "All master replication stream processed, " "manual failover can start."); + clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_FAILOVER); + return; } + clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_MANUALFAILOVER); } /* ----------------------------------------------------------------------------- @@ -3774,25 +3777,35 @@ void clusterCron(void) { * handlers, or to perform potentially expansive tasks that we need to do * a single time before replying to clients. */ void clusterBeforeSleep(void) { - /* Handle failover, this is needed when it is likely that there is already - * the quorum from masters in order to react fast. */ - if (g_pserver->cluster->todo_before_sleep & CLUSTER_TODO_HANDLE_FAILOVER) - clusterHandleSlaveFailover(); - - /* Update the cluster state. */ - if (g_pserver->cluster->todo_before_sleep & CLUSTER_TODO_UPDATE_STATE) - clusterUpdateState(); - - /* Save the config, possibly using fsync. */ - if (g_pserver->cluster->todo_before_sleep & CLUSTER_TODO_SAVE_CONFIG) { - int fsync = g_pserver->cluster->todo_before_sleep & - CLUSTER_TODO_FSYNC_CONFIG; - clusterSaveConfigOrDie(fsync); - } + int flags = g_pserver->cluster->todo_before_sleep; /* Reset our flags (not strictly needed since every single function * called for flags set should be able to clear its flag). */ g_pserver->cluster->todo_before_sleep = 0; + + if (flags & CLUSTER_TODO_HANDLE_MANUALFAILOVER) { + /* Handle manual failover as soon as possible so that won't have a 100ms + * as it was handled only in clusterCron */ + if(nodeIsSlave(myself)) { + clusterHandleManualFailover(); + if (!(g_pserver->cluster_module_flags & CLUSTER_MODULE_FLAG_NO_FAILOVER)) + clusterHandleSlaveFailover(); + } + } else if (flags & CLUSTER_TODO_HANDLE_FAILOVER) { + /* Handle failover, this is needed when it is likely that there is already + * the quorum from masters in order to react fast. */ + clusterHandleSlaveFailover(); + } + + /* Update the cluster state. */ + if (flags & CLUSTER_TODO_UPDATE_STATE) + clusterUpdateState(); + + /* Save the config, possibly using fsync. */ + if (flags & CLUSTER_TODO_SAVE_CONFIG) { + int fsync = flags & CLUSTER_TODO_FSYNC_CONFIG; + clusterSaveConfigOrDie(fsync); + } } void clusterDoBeforeSleep(int flags) { @@ -4173,8 +4186,8 @@ sds clusterGenNodeDescription(clusterNode *node) { sds ci; /* Node coordinates */ - ci = sdscatprintf(sdsempty(),"%.40s %s:%d@%d ", - node->name, + ci = sdscatlen(sdsempty(),node->name,CLUSTER_NAMELEN); + ci = sdscatfmt(ci," %s:%i@%i ", node->ip, node->port, node->cport); @@ -4183,40 +4196,46 @@ sds clusterGenNodeDescription(clusterNode *node) { ci = representClusterNodeFlags(ci, node->flags); /* Slave of... or just "-" */ + ci = sdscatlen(ci," ",1); if (node->slaveof) - ci = sdscatprintf(ci," %.40s ",node->slaveof->name); + ci = sdscatlen(ci,node->slaveof->name,CLUSTER_NAMELEN); else - ci = sdscatlen(ci," - ",3); + ci = sdscatlen(ci,"-",1); unsigned long long nodeEpoch = node->configEpoch; if (nodeIsSlave(node) && node->slaveof) { nodeEpoch = node->slaveof->configEpoch; } /* Latency from the POV of this node, config epoch, link status */ - ci = sdscatprintf(ci,"%lld %lld %llu %s", + ci = sdscatfmt(ci," %I %I %U %s", (long long) node->ping_sent, (long long) node->pong_received, nodeEpoch, (node->link || node->flags & CLUSTER_NODE_MYSELF) ? "connected" : "disconnected"); - /* Slots served by this instance */ - start = -1; - for (j = 0; j < CLUSTER_SLOTS; j++) { - int bit; + /* Slots served by this instance. If we already have slots info, + * append it diretly, otherwise, generate slots only if it has. */ + if (node->slots_info) { + ci = sdscatsds(ci, node->slots_info); + } else if (node->numslots > 0) { + start = -1; + for (j = 0; j < CLUSTER_SLOTS; j++) { + int bit; - if ((bit = clusterNodeGetSlotBit(node,j)) != 0) { - if (start == -1) start = j; - } - if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) { - if (bit && j == CLUSTER_SLOTS-1) j++; - - if (start == j-1) { - ci = sdscatprintf(ci," %d",start); - } else { - ci = sdscatprintf(ci," %d-%d",start,j-1); + if ((bit = clusterNodeGetSlotBit(node,j)) != 0) { + if (start == -1) start = j; + } + if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) { + if (bit && j == CLUSTER_SLOTS-1) j++; + + if (start == j-1) { + ci = sdscatfmt(ci," %i",start); + } else { + ci = sdscatfmt(ci," %i-%i",start,j-1); + } + start = -1; } - start = -1; } } @@ -4237,6 +4256,41 @@ sds clusterGenNodeDescription(clusterNode *node) { return ci; } +/* Generate the slot topology for all nodes and store the string representation + * in the slots_info struct on the node. This is used to improve the efficiency + * of clusterGenNodesDescription() because it removes looping of the slot space + * for generating the slot info for each node individually. */ +void clusterGenNodesSlotsInfo(int filter) { + clusterNode *n = NULL; + int start = -1; + + for (int i = 0; i <= CLUSTER_SLOTS; i++) { + /* Find start node and slot id. */ + if (n == NULL) { + if (i == CLUSTER_SLOTS) break; + n = g_pserver->cluster->slots[i]; + start = i; + continue; + } + + /* Generate slots info when occur different node with start + * or end of slot. */ + if (i == CLUSTER_SLOTS || n != g_pserver->cluster->slots[i]) { + if (!(n->flags & filter)) { + if (n->slots_info == NULL) n->slots_info = sdsempty(); + if (start == i-1) { + n->slots_info = sdscatfmt(n->slots_info," %i",start); + } else { + n->slots_info = sdscatfmt(n->slots_info," %i-%i",start,i-1); + } + } + if (i == CLUSTER_SLOTS) break; + n = g_pserver->cluster->slots[i]; + start = i; + } + } +} + /* Generate a csv-alike representation of the nodes we are aware of, * including the "myself" node, and return an SDS string containing the * representation (it is up to the caller to free it). @@ -4254,6 +4308,9 @@ sds clusterGenNodesDescription(int filter) { dictIterator *di; dictEntry *de; + /* Generate all nodes slots info firstly. */ + clusterGenNodesSlotsInfo(filter); + di = dictGetSafeIterator(g_pserver->cluster->nodes); while((de = dictNext(di)) != NULL) { clusterNode *node = (clusterNode*)dictGetVal(de); @@ -4263,6 +4320,12 @@ sds clusterGenNodesDescription(int filter) { ci = sdscatsds(ci,ni); sdsfree(ni); ci = sdscatlen(ci,"\n",1); + + /* Release slots info. */ + if (node->slots_info) { + sdsfree(node->slots_info); + node->slots_info = NULL; + } } dictReleaseIterator(di); return ci; @@ -4385,28 +4448,49 @@ void clusterCommand(client *c) { if (c->argc == 2 && !strcasecmp(szFromObj(c->argv[1]),"help")) { const char *help[] = { -"ADDSLOTS [slot ...] -- Assign slots to current node.", -"BUMPEPOCH -- Advance the cluster config epoch.", -"COUNT-failure-reports -- Return number of failure reports for .", -"COUNTKEYSINSLOT - Return the number of keys in .", -"DELSLOTS [slot ...] -- Delete slots information from current node.", -"FAILOVER [force|takeover] -- Promote current replica node to being a master.", -"FORGET -- Remove a node from the cluster.", -"GETKEYSINSLOT -- Return key names stored by current node in a slot.", -"FLUSHSLOTS -- Delete current node own slots information.", -"INFO - Return information about the cluster.", -"KEYSLOT -- Return the hash slot for .", -"MEET [bus-port] -- Connect nodes into a working cluster.", -"MYID -- Return the node id.", -"NODES -- Return cluster configuration seen by node. Output format:", -" ... ", -"REPLICATE -- Configure current node as replica to .", -"RESET [hard|soft] -- Reset current node (default: soft).", -"SET-config-epoch - Set config epoch of current node.", -"SETSLOT (importing|migrating|stable|node ) -- Set slot state.", -"REPLICAS -- Return replicas.", -"SAVECONFIG - Force saving cluster configuration on disk.", -"SLOTS -- Return information about slots range mappings. Each range is made of:", +"ADDSLOTS [ ...]", +" Assign slots to current node.", +"BUMPEPOCH", +" Advance the cluster config epoch.", +"COUNT-FAILURE-REPORTS ", +" Return number of failure reports for .", +"COUNTKEYSINSLOT ", +" Return the number of keys in .", +"DELSLOTS [ ...]", +" Delete slots information from current node.", +"FAILOVER [FORCE|TAKEOVER]", +" Promote current replica node to being a master.", +"FORGET ", +" Remove a node from the cluster.", +"GETKEYSINSLOT ", +" Return key names stored by current node in a slot.", +"FLUSHSLOTS", +" Delete current node own slots information.", +"INFO", +" Return information about the cluster.", +"KEYSLOT ", +" Return the hash slot for .", +"MEET []", +" Connect nodes into a working cluster.", +"MYID", +" Return the node id.", +"NODES", +" Return cluster configuration seen by node. Output format:", +" ...", +"REPLICATE ", +" Configure current node as replica to .", +"RESET [HARD|SOFT]", +" Reset current node (default: soft).", +"SET-CONFIG-EPOCH ", +" Set config epoch of current node.", +"SETSLOT (IMPORTING|MIGRATING|STABLE|NODE )", +" Set slot state.", +"REPLICAS ", +" Return replicas.", +"SAVECONFIG", +" Force saving cluster configuration on disk.", +"SLOTS", +" Return information about slots range mappings. Each range is made of:", " start, end, master and replicas IP addresses, ports and ids", NULL }; @@ -4577,6 +4661,9 @@ NULL g_pserver->cluster->migrating_slots_to[slot]) g_pserver->cluster->migrating_slots_to[slot] = NULL; + clusterDelSlot(slot); + clusterAddSlot(n,slot); + /* If this node was importing this slot, assigning the slot to * itself also clears the importing status. */ if (n == myself && @@ -4596,9 +4683,10 @@ NULL "configEpoch updated after importing slot %d", slot); } g_pserver->cluster->importing_slots_from[slot] = NULL; + /* After importing this slot, let the other nodes know as + * soon as possible. */ + clusterBroadcastPong(CLUSTER_BROADCAST_ALL); } - clusterDelSlot(slot); - clusterAddSlot(n,slot); } else { addReplyError(c, "Invalid CLUSTER SETSLOT action or number of arguments. Try CLUSTER HELP"); @@ -4844,7 +4932,7 @@ NULL takeover = 1; force = 1; /* Takeover also implies force. */ } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } @@ -4935,7 +5023,7 @@ NULL } else if (!strcasecmp(szFromObj(c->argv[2]),"soft")) { hard = 0; } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } @@ -5007,6 +5095,9 @@ int verifyDumpPayload(unsigned char *p, size_t len) { rdbver = (footer[1] << 8) | footer[0]; if (rdbver > RDB_VERSION) return C_ERR; + if (cserver.skip_checksum_validation) + return C_OK; + /* Verify CRC64 */ crc = crc64(0,p,len-8); memrev64ifbe(&crc); @@ -5066,7 +5157,7 @@ void mvccrestoreCommand(client *c) { setMvccTstamp(obj, mvcc); /* Create the key and set the TTL if any */ - if (dbMerge(c->db,key,obj,true)) { + if (dbMerge(c->db,szFromObj(key),obj,true)) { if (expire >= 0) { setExpire(c,c->db,key,nullptr,expire); } @@ -5115,7 +5206,7 @@ void restoreCommand(client *c) { } j++; /* Consume additional arg. */ } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } @@ -5123,7 +5214,7 @@ void restoreCommand(client *c) { /* Make sure this key does not already exist here... */ robj *key = c->argv[1]; if (!replace && lookupKeyWrite(c->db,key) != NULL) { - addReply(c,shared.busykeyerr); + addReplyErrorObject(c,shared.busykeyerr); return; } @@ -5236,8 +5327,7 @@ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long ti conn = g_pserver->tls_cluster ? connCreateTLS() : connCreateSocket(); if (connBlockingConnect(conn, szFromObj(c->argv[1]), atoi(szFromObj(c->argv[2])), timeout) != C_OK) { - addReplySds(c, - sdsnew("-IOERR error or timeout connecting to the client\r\n")); + addReplyError(c,"-IOERR error or timeout connecting to the client"); connClose(conn); sdsfree(name); return NULL; @@ -5324,7 +5414,7 @@ void migrateCommand(client *c) { replace = 1; } else if (!strcasecmp(szFromObj(c->argv[j]),"auth")) { if (!moreargs) { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } j++; @@ -5347,7 +5437,7 @@ void migrateCommand(client *c) { num_keys = c->argc - j - 1; break; /* All the remaining args are keys. */ } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } @@ -5840,7 +5930,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in * cluster is down. */ if (error_code) *error_code = CLUSTER_REDIR_DOWN_STATE; return NULL; - } else if (!(cmd->flags & CMD_READONLY) && !(cmd->proc == evalCommand) + } else if ((cmd->flags & CMD_WRITE) && !(cmd->proc == evalCommand) && !(cmd->proc == evalShaCommand)) { /* The cluster is configured to allow read only commands @@ -5889,11 +5979,10 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in /* Handle the read-only client case reading from a slave: if this * node is a slave and the request is about a hash slot our master * is serving, we can reply without redirection. */ - int is_readonly_command = (c->cmd->flags & CMD_READONLY) || - (c->cmd->proc == execCommand && !(c->mstate.cmd_inv_flags & CMD_READONLY)); + int is_write_command = (c->cmd->flags & CMD_WRITE) || + (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_WRITE)); if (c->flags & CLIENT_READONLY && - (is_readonly_command || cmd->proc == evalCommand || - cmd->proc == evalShaCommand) && + (!is_write_command || cmd->proc == evalCommand || cmd->proc == evalShaCommand) && nodeIsSlave(myself) && myself->slaveof == n) { @@ -5915,23 +6004,23 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in * be set to the hash slot that caused the redirection. */ void clusterRedirectClient(client *c, clusterNode *n, int hashslot, int error_code) { if (error_code == CLUSTER_REDIR_CROSS_SLOT) { - addReplySds(c,sdsnew("-CROSSSLOT Keys in request don't hash to the same slot\r\n")); + addReplyError(c,"-CROSSSLOT Keys in request don't hash to the same slot"); } else if (error_code == CLUSTER_REDIR_UNSTABLE) { /* The request spawns multiple keys in the same slot, * but the slot is not "stable" currently as there is * a migration or import in progress. */ - addReplySds(c,sdsnew("-TRYAGAIN Multiple keys request during rehashing of slot\r\n")); + addReplyError(c,"-TRYAGAIN Multiple keys request during rehashing of slot"); } else if (error_code == CLUSTER_REDIR_DOWN_STATE) { - addReplySds(c,sdsnew("-CLUSTERDOWN The cluster is down\r\n")); + addReplyError(c,"-CLUSTERDOWN The cluster is down"); } else if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) { - addReplySds(c,sdsnew("-CLUSTERDOWN The cluster is down and only accepts read commands\r\n")); + addReplyError(c,"-CLUSTERDOWN The cluster is down and only accepts read commands"); } else if (error_code == CLUSTER_REDIR_DOWN_UNBOUND) { - addReplySds(c,sdsnew("-CLUSTERDOWN Hash slot not served\r\n")); + addReplyError(c,"-CLUSTERDOWN Hash slot not served"); } else if (error_code == CLUSTER_REDIR_MOVED || error_code == CLUSTER_REDIR_ASK) { - addReplySds(c,sdscatprintf(sdsempty(), - "-%s %d %s:%d\r\n", + addReplyErrorSds(c,sdscatprintf(sdsempty(), + "-%s %d %s:%d", (error_code == CLUSTER_REDIR_ASK) ? "ASK" : "MOVED", hashslot,n->ip,n->port)); } else { @@ -5985,6 +6074,15 @@ int clusterRedirectBlockedClientIfNeeded(client *c) { node = myself; } + /* if the client is read-only and attempting to access key that our + * replica can handle, allow it. */ + if ((c->flags & CLIENT_READONLY) && + !(c->lastcmd->flags & CMD_WRITE) && + nodeIsSlave(myself) && myself->slaveof == node) + { + node = myself; + } + /* We send an error and unblock the client if: * 1) The slot is unassigned, emitting a cluster down error. * 2) The slot is not handled by this node, nor being imported. */ diff --git a/src/cluster.h b/src/cluster.h index 6dfe318e2..9d370b360 100644 --- a/src/cluster.h +++ b/src/cluster.h @@ -83,6 +83,7 @@ typedef struct clusterLink { #define CLUSTER_TODO_UPDATE_STATE (1<<1) #define CLUSTER_TODO_SAVE_CONFIG (1<<2) #define CLUSTER_TODO_FSYNC_CONFIG (1<<3) +#define CLUSTER_TODO_HANDLE_MANUALFAILOVER (1<<4) /* Message types. * @@ -121,6 +122,7 @@ typedef struct clusterNode { int flags; /* CLUSTER_NODE_... */ uint64_t configEpoch; /* Last configEpoch observed for this node */ unsigned char slots[CLUSTER_SLOTS/8]; /* slots handled by this node */ + sds slots_info; /* Slots info represented by string. */ int numslots; /* Number of slots handled by this node */ int numslaves; /* Number of slave nodes, if this is a master */ struct clusterNode **slaves; /* pointers to slave nodes */ diff --git a/src/compactvector.h b/src/compactvector.h index daa8ad9fc..bed1659d6 100644 --- a/src/compactvector.h +++ b/src/compactvector.h @@ -73,7 +73,7 @@ public: m_max = m_max + 4; m_data = (T*)zrealloc(m_data, sizeof(T) * m_max, MALLOC_LOCAL); - m_max = zmalloc_usable(m_data) / sizeof(T); + m_max = zmalloc_usable_size(m_data) / sizeof(T); } assert(idx < m_max); where = m_data + idx; diff --git a/src/config.cpp b/src/config.cpp index f128f40ed..035162b73 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -115,6 +115,19 @@ configEnum oom_score_adj_enum[] = { {NULL, 0} }; +configEnum acl_pubsub_default_enum[] = { + {"allchannels", USER_FLAG_ALLCHANNELS}, + {"resetchannels", 0}, + {NULL, 0} +}; + +configEnum sanitize_dump_payload_enum[] = { + {"no", SANITIZE_DUMP_NO}, + {"yes", SANITIZE_DUMP_YES}, + {"clients", SANITIZE_DUMP_CLIENTS}, + {NULL, 0} +}; + /* Output buffer limits presets. */ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { {0, 0, 0}, /* normal */ @@ -155,6 +168,15 @@ typedef struct stringConfigData { be stored as a NULL value. */ } stringConfigData; +typedef struct sdsConfigData { + sds *config; /* Pointer to the server config this value is stored in. */ + const char *default_value; /* Default value of the config on rewrite. */ + int (*is_valid_fn)(sds val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(sds val, sds prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ + int convert_empty_to_null; /* Boolean indicating if empty SDS strings should + be stored as a NULL value. */ +} sdsConfigData; + typedef struct enumConfigData { int *config; /* The pointer to the server config this value is stored in */ configEnum *enum_value; /* The underlying enum type this data represents */ @@ -201,6 +223,7 @@ typedef struct numericConfigData { typedef union typeData { boolConfigData yesno; stringConfigData string; + sdsConfigData sds; enumConfigData enumd; numericConfigData numeric; } typeData; @@ -375,6 +398,7 @@ void loadServerConfigFromString(char *config) { int linenum = 0, totlines, i; int slaveof_linenum = 0; sds *lines; + int save_loaded = 0; lines = sdssplitlen(config,strlen(config),"\n",1,&totlines); @@ -447,6 +471,14 @@ void loadServerConfigFromString(char *config) { err = "Invalid socket file permissions"; goto loaderr; } } else if (!strcasecmp(argv[0],"save")) { + /* We don't reset save params before loading, because if they're not part + * of the file the defaults should be used. + */ + if (!save_loaded) { + save_loaded = 1; + resetServerSaveParams(); + } + if (argc == 3) { int seconds = atoi(argv[1]); int changes = atoi(argv[2]); @@ -480,9 +512,7 @@ void loadServerConfigFromString(char *config) { fclose(logfp); } } else if (!strcasecmp(argv[0],"include") && argc == 2) { - loadServerConfig(argv[1],NULL); - } else if ((!strcasecmp(argv[0],"client-query-buffer-limit")) && argc == 2) { - cserver.client_max_querybuf_len = memtoll(argv[1],NULL); + loadServerConfig(argv[1], 0, NULL); } else if ((!strcasecmp(argv[0],"slaveof") || !strcasecmp(argv[0],"replicaof")) && argc == 3) { slaveof_linenum = linenum; @@ -494,7 +524,7 @@ void loadServerConfigFromString(char *config) { while ((ln = listNext(&li))) { struct redisMaster *mi = (struct redisMaster*)listNodeValue(ln); - zfree(mi->masterauth); + sdsfree(mi->masterauth); zfree(mi->masteruser); zfree(mi->repl_transfer_tmpfile); delete mi->staleKeyMap; @@ -515,21 +545,6 @@ void loadServerConfigFromString(char *config) { err = "Password is longer than CONFIG_AUTHPASS_MAX_LEN"; goto loaderr; } - /* The old "requirepass" directive just translates to setting - * a password to the default user. The only thing we do - * additionally is to remember the cleartext password in this - * case, for backward compatibility with Redis <= 5. */ - ACLSetUser(DefaultUser,"resetpass",-1); - sdsfree(g_pserver->requirepass); - g_pserver->requirepass = NULL; - if (sdslen(argv[1])) { - sds aclop = sdscatprintf(sdsempty(),">%s",argv[1]); - ACLSetUser(DefaultUser,aclop,sdslen(aclop)); - sdsfree(aclop); - g_pserver->requirepass = sdsnew(argv[1]); - } else { - ACLSetUser(DefaultUser,"nopass",-1); - } } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){ /* DEAD OPTION */ } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) { @@ -613,8 +628,7 @@ void loadServerConfigFromString(char *config) { err = "sentinel directive while not in sentinel mode"; goto loaderr; } - err = sentinelHandleConfiguration(argv+1,argc-1); - if (err) goto loaderr; + queueSentinelConfig(argv+1,argc-1,linenum,lines[i]); } } else if (!strcasecmp(argv[0],"scratch-file-path")) { #ifdef USE_MEMKIND @@ -697,28 +711,31 @@ loaderr: * Both filename and options can be NULL, in such a case are considered * empty. This way loadServerConfig can be used to just load a file or * just load a string. */ -void loadServerConfig(char *filename, char *options) { +void loadServerConfig(char *filename, char config_from_stdin, char *options) { sds config = sdsempty(); char buf[CONFIG_MAX_LINE+1]; + FILE *fp; /* Load the file content */ if (filename) { - FILE *fp; - - if (filename[0] == '-' && filename[1] == '\0') { - fp = stdin; - } else { - if ((fp = fopen(filename,"r")) == NULL) { - serverLog(LL_WARNING, + if ((fp = fopen(filename,"r")) == NULL) { + serverLog(LL_WARNING, "Fatal error, can't open config file '%s': %s", filename, strerror(errno)); - exit(1); - } + exit(1); } while(fgets(buf,CONFIG_MAX_LINE+1,fp) != NULL) config = sdscat(config,buf); - if (fp != stdin) fclose(fp); + fclose(fp); } + /* Append content from stdin */ + if (config_from_stdin) { + serverLog(LL_WARNING,"Reading config from stdin"); + fp = stdin; + while(fgets(buf,CONFIG_MAX_LINE+1,fp) != NULL) + config = sdscat(config,buf); + } + /* Append the additional options */ if (options) { config = sdscat(config,"\n"); @@ -785,23 +802,38 @@ void configSetCommand(client *c) { if (0) { /* this starts the config_set macros else-if chain. */ /* Special fields that can't be handled with general macros. */ - config_set_special_field("requirepass") { - if (sdslen(szFromObj(o)) > CONFIG_AUTHPASS_MAX_LEN) goto badfmt; - /* The old "requirepass" directive just translates to setting - * a password to the default user. The only thing we do - * additionally is to remember the cleartext password in this - * case, for backward compatibility with Redis <= 5. */ - ACLSetUser(DefaultUser,"resetpass",-1); - sdsfree(g_pserver->requirepass); - g_pserver->requirepass = NULL; - if (sdslen(szFromObj(o))) { - sds aclop = sdscatprintf(sdsempty(),">%s",(char*)ptrFromObj(o)); - ACLSetUser(DefaultUser,aclop,sdslen(aclop)); - sdsfree(aclop); - g_pserver->requirepass = sdsnew(szFromObj(o)); - } else { - ACLSetUser(DefaultUser,"nopass",-1); + config_set_special_field("bind") { + int vlen; + sds *v = sdssplitlen(szFromObj(o),sdslen(szFromObj(o))," ",1,&vlen); + + if (vlen < 1 || vlen > CONFIG_BINDADDR_MAX) { + addReplyError(c, "Too many bind addresses specified."); + sdsfreesplitres(v, vlen); + return; } + + if (changeBindAddr(v, vlen, true) == C_ERR) { + addReplyError(c, "Failed to bind to specified addresses."); + sdsfreesplitres(v, vlen); + return; + } + // Now run the config change on the other threads + for (int ithread = 0; ithread < cserver.cthreads; ++ithread) { + if (&g_pserver->rgthreadvar[ithread] != serverTL) { + incrRefCount(o); + aePostFunction(g_pserver->rgthreadvar[ithread].el, [o]{ + int vlen; + sds *v = sdssplitlen(szFromObj(o),sdslen(szFromObj(o))," ",1,&vlen); + if (changeBindAddr(v, vlen, false) == C_ERR) { + serverLog(LL_WARNING, "Failed to change the bind address for a thread. Server will still be listening on old addresses."); + } + sdsfreesplitres(v, vlen); + decrRefCount(o); + }); + } + } + + sdsfreesplitres(v, vlen); } config_set_special_field("save") { int vlen, j; sds *v = sdssplitlen(szFromObj(o),sdslen(szFromObj(o))," ",1,&vlen); @@ -910,10 +942,6 @@ void configSetCommand(client *c) { enableWatchdog(ll); else disableWatchdog(); - /* Memory fields. - * config_set_memory_field(name,var) */ - } config_set_memory_field( - "client-query-buffer-limit",cserver.client_max_querybuf_len) { /* Everything else is an error... */ } config_set_else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", @@ -948,7 +976,7 @@ badfmt: /* Bad format errors */ addReplyBulkCString(c,_var ? _var : ""); \ matches++; \ } \ -} while(0); +} while(0) #define config_get_bool_field(_name,_var) do { \ if (stringmatch(pattern,_name,1)) { \ @@ -956,7 +984,7 @@ badfmt: /* Bad format errors */ addReplyBulkCString(c,_var ? "yes" : "no"); \ matches++; \ } \ -} while(0); +} while(0) #define config_get_numerical_field(_name,_var) do { \ if (stringmatch(pattern,_name,1)) { \ @@ -965,8 +993,7 @@ badfmt: /* Bad format errors */ addReplyBulkCString(c,buf); \ matches++; \ } \ -} while(0); - +} while(0) void configGetCommand(client *c) { robj *o = c->argv[2]; @@ -994,7 +1021,6 @@ void configGetCommand(client *c) { config_get_string_field("logfile",g_pserver->logfile); /* Numerical values */ - config_get_numerical_field("client-query-buffer-limit",cserver.client_max_querybuf_len); config_get_numerical_field("watchdog-period",g_pserver->watchdog_period); /* Everything we can't handle with macros follows. */ @@ -1045,7 +1071,7 @@ void configGetCommand(client *c) { } if (stringmatch(pattern,"unixsocketperm",1)) { char buf[32]; - snprintf(buf,sizeof(buf),"%o",g_pserver->unixsocketperm); + snprintf(buf,sizeof(buf),"%lo",(unsigned long)g_pserver->unixsocketperm); addReplyBulkCString(c,"unixsocketperm"); addReplyBulkCString(c,buf); matches++; @@ -1099,16 +1125,6 @@ void configGetCommand(client *c) { sdsfree(aux); matches++; } - if (stringmatch(pattern,"requirepass",1)) { - addReplyBulkCString(c,"requirepass"); - sds password = g_pserver->requirepass; - if (password) { - addReplyBulkCBuffer(c,password,sdslen(password)); - } else { - addReplyBulkCString(c,""); - } - matches++; - } if (stringmatch(pattern,"oom-score-adj-values",0)) { sds buf = sdsempty(); int j; @@ -1157,7 +1173,8 @@ dictType optionToLineDictType = { NULL, /* val dup */ dictSdsKeyCaseCompare, /* key compare */ dictSdsDestructor, /* key destructor */ - dictListDestructor /* val destructor */ + dictListDestructor, /* val destructor */ + NULL /* allow to expand */ }; dictType optionSetDictType = { @@ -1166,7 +1183,8 @@ dictType optionSetDictType = { NULL, /* val dup */ dictSdsKeyCaseCompare, /* key compare */ dictSdsDestructor, /* key destructor */ - NULL /* val destructor */ + NULL, /* val destructor */ + NULL /* allow to expand */ }; /* The config rewrite state. */ @@ -1270,13 +1288,22 @@ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) { char *p = strstr(argv[0],"slave"); if (p) { sds alt = sdsempty(); - alt = sdscatlen(alt,argv[0],p-argv[0]);; + alt = sdscatlen(alt,argv[0],p-argv[0]); alt = sdscatlen(alt,"replica",7); alt = sdscatlen(alt,p+5,strlen(p+5)); sdsfree(argv[0]); argv[0] = alt; } - rewriteConfigAddLineNumberToOption(state,argv[0],linenum); + /* If this is sentinel config, we use sentinel "sentinel " as option + to avoid messing up the sequence. */ + if (g_pserver->sentinel_mode && argc > 1 && !strcasecmp(argv[0],"sentinel")) { + sds sentinelOption = sdsempty(); + sentinelOption = sdscatfmt(sentinelOption,"%S %S",argv[0],argv[1]); + rewriteConfigAddLineNumberToOption(state,sentinelOption,linenum); + sdsfree(sentinelOption); + } else { + rewriteConfigAddLineNumberToOption(state,argv[0],linenum); + } sdsfreesplitres(argv,argc); } fclose(fp); @@ -1394,6 +1421,28 @@ void rewriteConfigStringOption(struct rewriteConfigState *state, const char *opt rewriteConfigRewriteLine(state,option,line,force); } +/* Rewrite a SDS string option. */ +void rewriteConfigSdsOption(struct rewriteConfigState *state, const char *option, sds value, const sds defvalue) { + int force = 1; + sds line; + + /* If there is no value set, we don't want the SDS option + * to be present in the configuration at all. */ + if (value == NULL) { + rewriteConfigMarkAsProcessed(state, option); + return; + } + + /* Set force to zero if the value is set to its default. */ + if (defvalue && sdscmp(value, defvalue) == 0) force = 0; + + line = sdsnew(option); + line = sdscatlen(line, " ", 1); + line = sdscatrepr(line, value, sdslen(value)); + + rewriteConfigRewriteLine(state, option, line, force); +} + /* Rewrite a numerical (long long range) option. */ void rewriteConfigNumericalOption(struct rewriteConfigState *state, const char *option, long long value, long long defvalue) { int force = value != defvalue; @@ -1597,26 +1646,6 @@ void rewriteConfigBindOption(struct rewriteConfigState *state) { rewriteConfigRewriteLine(state,option,line,force); } -/* Rewrite the requirepass option. */ -void rewriteConfigRequirepassOption(struct rewriteConfigState *state, const char *option) { - int force = 1; - sds line; - sds password = g_pserver->requirepass; - - /* If there is no password set, we don't want the requirepass option - * to be present in the configuration at all. */ - if (password == NULL) { - rewriteConfigMarkAsProcessed(state,option); - return; - } - - line = sdsnew(option); - line = sdscatlen(line, " ", 1); - line = sdscatsds(line, password); - - rewriteConfigRewriteLine(state,option,line,force); -} - /* Glue together the configuration lines in the current configuration * rewrite state into a single string, stripping multiple empty lines. */ sds rewriteConfigGetContentFromState(struct rewriteConfigState *state) { @@ -1724,7 +1753,7 @@ int rewriteConfigOverwriteFile(char *configfile, sds content) { if (fsync(fd)) serverLog(LL_WARNING, "Could not sync tmp config file to disk (%s)", strerror(errno)); - else if (fchmod(fd, 0644) == -1) + else if (fchmod(fd, 0644 & ~g_pserver->umask) == -1) serverLog(LL_WARNING, "Could not chmod config file (%s)", strerror(errno)); else if (rename(tmp_conffile, configfile) == -1) serverLog(LL_WARNING, "Could not rename tmp config file (%s)", strerror(errno)); @@ -1773,8 +1802,6 @@ int rewriteConfig(char *path, int force_all) { rewriteConfigUserOption(state); rewriteConfigDirOption(state); rewriteConfigSlaveofOption(state,"replicaof"); - rewriteConfigRequirepassOption(state,"requirepass"); - rewriteConfigBytesOption(state,"client-query-buffer-limit",cserver.client_max_querybuf_len,PROTO_MAX_QUERYBUF_LEN); rewriteConfigStringOption(state,"cluster-config-file",g_pserver->cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); rewriteConfigNotifykeyspaceeventsOption(state); rewriteConfigClientoutputbufferlimitOption(state); @@ -1875,22 +1902,14 @@ constexpr standardConfig createBoolConfig(const char *name, const char *alias, i /* String Configs */ static void stringConfigInit(typeData data) { - if (data.string.convert_empty_to_null) { - *data.string.config = data.string.default_value ? zstrdup(data.string.default_value) : NULL; - } else { - *data.string.config = zstrdup(data.string.default_value); - } + *data.string.config = (data.string.convert_empty_to_null && !data.string.default_value) ? NULL : zstrdup(data.string.default_value); } static int stringConfigSet(typeData data, sds value, int update, const char **err) { if (data.string.is_valid_fn && !data.string.is_valid_fn(value, err)) return 0; char *prev = *data.string.config; - if (data.string.convert_empty_to_null) { - *data.string.config = value[0] ? zstrdup(value) : NULL; - } else { - *data.string.config = zstrdup(value); - } + *data.string.config = (data.string.convert_empty_to_null && !value[0]) ? NULL : zstrdup(value); if (update && data.string.update_fn && !data.string.update_fn(*data.string.config, prev, err)) { zfree(*data.string.config); *data.string.config = prev; @@ -1908,6 +1927,38 @@ static void stringConfigRewrite(typeData data, const char *name, struct rewriteC rewriteConfigStringOption(state, name,*(data.string.config), data.string.default_value); } +/* SDS Configs */ +static void sdsConfigInit(typeData data) { + *data.sds.config = (data.sds.convert_empty_to_null && !data.sds.default_value) ? NULL: sdsnew(data.sds.default_value); +} + +static int sdsConfigSet(typeData data, sds value, int update, const char **err) { + if (data.sds.is_valid_fn && !data.sds.is_valid_fn(value, err)) + return 0; + sds prev = *data.sds.config; + *data.sds.config = (data.sds.convert_empty_to_null && (sdslen(value) == 0)) ? NULL : sdsdup(value); + if (update && data.sds.update_fn && !data.sds.update_fn(*data.sds.config, prev, err)) { + sdsfree(*data.sds.config); + *data.sds.config = prev; + return 0; + } + sdsfree(prev); + return 1; +} + +static void sdsConfigGet(client *c, typeData data) { + if (*data.sds.config) { + addReplyBulkSds(c, sdsdup(*data.sds.config)); + } else { + addReplyBulkCString(c, ""); + } +} + +static void sdsConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigSdsOption(state, name, *(data.sds.config), data.sds.default_value ? sdsnew(data.sds.default_value) : NULL); +} + + #define ALLOW_EMPTY_STRING 0 #define EMPTY_STRING_IS_NULL 1 @@ -1926,6 +1977,21 @@ constexpr standardConfig createStringConfig(const char *name, const char *alias, return conf; } +constexpr standardConfig createSDSConfig(const char *name, const char *alias, int modifiable, int empty_to_null, sds &config_addr, const char *defaultValue, int (*is_valid)(char*,const char**), int (*update)(char*,char*,const char**)) { + standardConfig conf = { + embedCommonConfig(name, alias, modifiable) + embedConfigInterface(sdsConfigInit, sdsConfigSet, sdsConfigGet, sdsConfigRewrite) + }; + conf.data.sds = { + &(config_addr), + (defaultValue), + (is_valid), + (update), + (empty_to_null), + }; + return conf; +} + /* Enum configs */ static void enumConfigInit(typeData data) { *data.enumd.config = data.enumd.default_value; @@ -1944,7 +2010,7 @@ static int enumConfigSet(typeData data, sds value, int update, const char **err) } sdsrange(enumerr,0,-3); /* Remove final ", ". */ - strncpy(loadbuf, enumerr, LOADBUF_SIZE-1); + strncpy(loadbuf, enumerr, LOADBUF_SIZE); loadbuf[LOADBUF_SIZE - 1] = '\0'; sdsfree(enumerr); @@ -2246,6 +2312,25 @@ static int isValidAOFfilename(char *val, const char **err) { return 1; } +/* Validate specified string is a valid proc-title-template */ +static int isValidProcTitleTemplate(char *val, const char **err) { + if (!validateProcTitleTemplate(val)) { + *err = "template format is invalid or contains unknown variables"; + return 0; + } + return 1; +} + +static int updateProcTitleTemplate(char *val, char *prev, const char **err) { + UNUSED(val); + UNUSED(prev); + if (redisSetProcTitle(NULL) == C_ERR) { + *err = "failed to set process title"; + return 0; + } + return 1; +} + static int updateHZ(long long val, long long prev, const char **err) { UNUSED(prev); UNUSED(err); @@ -2258,6 +2343,32 @@ static int updateHZ(long long val, long long prev, const char **err) { return 1; } +static int updatePort(long long val, long long prev, const char **err) { + /* Do nothing if port is unchanged */ + if (val == prev) { + return 1; + } + + // Run this thread to make sure its valid + if (changeListenPort(val, &serverTL->ipfd, acceptTcpHandler, true) == C_ERR) { + *err = "Unable to listen on this port. Check server logs."; + return 0; + } + + // Now run the config change on the other threads + for (int ithread = 0; ithread < cserver.cthreads; ++ithread) { + if (&g_pserver->rgthreadvar[ithread] != serverTL) { + aePostFunction(g_pserver->rgthreadvar[ithread].el, [val]{ + if (changeListenPort(val, &serverTL->ipfd, acceptTcpHandler, false) == C_ERR) { + serverLog(LL_WARNING, "Failed to change the listen port for a thread. Server will still be listening on old ports."); + } + }); + } + } + + return 1; +} + static int updateJemallocBgThread(int val, int prev, const char **err) { UNUSED(prev); UNUSED(err); @@ -2282,7 +2393,7 @@ static int updateMaxmemory(long long val, long long prev, const char **err) { if ((unsigned long long)val < used) { serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET (%llu) is smaller than the current memory usage (%zu). This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy.", g_pserver->maxmemory, used); } - freeMemoryIfNeededAndSafe(); + performEvictions(); } return 1; } @@ -2295,7 +2406,7 @@ static int updateGoodSlaves(long long val, long long prev, const char **err) { return 1; } -static int updateMasterAuthConfig(char *, char *, const char **) { +static int updateMasterAuthConfig(sds, sds, const char **) { updateMasterAuth(); return 1; } @@ -2313,6 +2424,16 @@ static int updateAppendonly(int val, int prev, const char **err) { return 1; } +static int updateSighandlerEnabled(int val, int prev, const char **err) { + UNUSED(err); + UNUSED(prev); + if (val) + setupSignalHandlers(); + else + removeSignalHandlers(); + return 1; +} + static int updateMaxclients(long long val, long long prev, const char **err) { /* Try to check if the OS is capable of supporting so many FDs. */ if (val > prev) { @@ -2389,6 +2510,17 @@ static int updateOOMScoreAdj(int val, int prev, const char **err) { return 1; } +int updateRequirePass(sds val, sds prev, const char **err) { + UNUSED(prev); + UNUSED(err); + /* The old "requirepass" directive just translates to setting + * a password to the default user. The only thing we do + * additionally is to remember the cleartext password in this + * case, for backward compatibility with Redis <= 5. */ + ACLUpdateDefaultUserPassword(val); + return 1; +} + #ifdef USE_OPENSSL static int updateTlsCfg(char *val, char *prev, const char **err) { UNUSED(val); @@ -2414,6 +2546,27 @@ static int updateTlsCfgInt(long long val, long long prev, const char **err) { UNUSED(prev); return updateTlsCfg(NULL, NULL, err); } + +static int updateTLSPort(long long val, long long prev, const char **err) { + /* Do nothing if port is unchanged */ + if (val == prev) { + return 1; + } + + /* Configure TLS if tls is enabled */ + if (prev == 0 && tlsConfigure(&server.tls_ctx_config) == C_ERR) { + *err = "Unable to update TLS configuration. Check server logs."; + return 0; + } +asdsa + if (changeListenPort(val, &server.tlsfd, acceptTLSHandler) == C_ERR) { + *err = "Unable to listen on this port. Check server logs."; + return 0; + } + + return 1; +} + #endif /* USE_OPENSSL */ int fDummy = false; @@ -2429,11 +2582,13 @@ standardConfig configs[] = { createBoolConfig("rdb-del-sync-files", NULL, MODIFIABLE_CONFIG, g_pserver->rdb_del_sync_files, 0, NULL, NULL), createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, g_pserver->activerehashing, 1, NULL, NULL), createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, g_pserver->stop_writes_on_bgsave_err, 1, NULL, NULL), + createBoolConfig("set-proc-title", NULL, IMMUTABLE_CONFIG, cserver.set_proc_title, 1, NULL, NULL), /* Should setproctitle be used? */ createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, g_pserver->dynamic_hz, 1, NULL, NULL), /* Adapt hz to # of clients.*/ createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, g_pserver->lazyfree_lazy_eviction, 0, NULL, NULL), createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, g_pserver->lazyfree_lazy_expire, 0, NULL, NULL), createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, g_pserver->lazyfree_lazy_server_del, 0, NULL, NULL), createBoolConfig("lazyfree-lazy-user-del", NULL, MODIFIABLE_CONFIG, g_pserver->lazyfree_lazy_user_del , 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-user-flush", NULL, MODIFIABLE_CONFIG, g_pserver->lazyfree_lazy_user_flush , 0, NULL, NULL), createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, g_pserver->repl_disable_tcp_nodelay, 0, NULL, NULL), createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, g_pserver->repl_diskless_sync, 0, NULL, NULL), createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, g_pserver->aof_rewrite_incremental_fsync, 1, NULL, NULL), @@ -2454,9 +2609,11 @@ standardConfig configs[] = { createBoolConfig("cluster-enabled", NULL, IMMUTABLE_CONFIG, g_pserver->cluster_enabled, 0, NULL, NULL), createBoolConfig("appendonly", NULL, MODIFIABLE_CONFIG, g_pserver->aof_enabled, 0, NULL, updateAppendonly), createBoolConfig("cluster-allow-reads-when-down", NULL, MODIFIABLE_CONFIG, g_pserver->cluster_allow_reads_when_down, 0, NULL, NULL), - createBoolConfig("multi-master-no-forward", NULL, MODIFIABLE_CONFIG, cserver.multimaster_no_forward, 0, validateMultiMasterNoForward, NULL), - createBoolConfig("allow-write-during-load", NULL, MODIFIABLE_CONFIG, g_pserver->fWriteDuringActiveLoad, 0, NULL, NULL), createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, fDummy, 0, NULL, NULL), + createBoolConfig("crash-log-enabled", NULL, MODIFIABLE_CONFIG, g_pserver->crashlog_enabled, 1, NULL, updateSighandlerEnabled), + createBoolConfig("crash-memcheck-enabled", NULL, MODIFIABLE_CONFIG, g_pserver->memcheck_enabled, 1, NULL, NULL), + createBoolConfig("use-exit-on-panic", NULL, MODIFIABLE_CONFIG, g_pserver->use_exit_on_panic, 0, NULL, NULL), + createBoolConfig("disable-thp", NULL, MODIFIABLE_CONFIG, g_pserver->disable_thp, 1, NULL, NULL), /* String Configs */ createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->acl_filename, "", NULL, NULL), @@ -2464,7 +2621,6 @@ standardConfig configs[] = { createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.pidfile, NULL, NULL, NULL), createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->slave_announce_ip, NULL, NULL, NULL), createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.default_masteruser, NULL, NULL, updateMasterAuthConfig), - createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.default_masterauth, NULL, NULL, updateMasterAuthConfig), createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->cluster_announce_ip, NULL, NULL, NULL), createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->syslog_ident, "redis", NULL, NULL), createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->rdb_filename, CONFIG_DEFAULT_RDB_FILENAME, isValidDBfilename, NULL), @@ -2473,7 +2629,12 @@ standardConfig configs[] = { createStringConfig("bio_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->bio_cpulist, NULL, NULL, NULL), createStringConfig("aof_rewrite_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->aof_rewrite_cpulist, NULL, NULL, NULL), createStringConfig("bgsave_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->bgsave_cpulist, NULL, NULL, NULL), - createStringConfig("ignore-warnings", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->ignore_warnings, "ARM64-COW-BUG", NULL, NULL), + createStringConfig("ignore-warnings", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->ignore_warnings, "", NULL, NULL), + createStringConfig("proc-title-template", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, cserver.proc_title_template, CONFIG_DEFAULT_PROC_TITLE_TEMPLATE, isValidProcTitleTemplate, updateProcTitleTemplate), + + /* SDS Configs */ + createSDSConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.default_masterauth, NULL, NULL, updateMasterAuthConfig), + createSDSConfig("requirepass", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->requirepass, NULL, NULL, updateRequirePass), /* Enum Configs */ createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, cserver.supervised_mode, SUPERVISED_NONE, NULL, NULL), @@ -2483,10 +2644,12 @@ standardConfig configs[] = { createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, g_pserver->maxmemory_policy, MAXMEMORY_NO_EVICTION, NULL, NULL), createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, g_pserver->aof_fsync, AOF_FSYNC_EVERYSEC, NULL, NULL), createEnumConfig("oom-score-adj", NULL, MODIFIABLE_CONFIG, oom_score_adj_enum, g_pserver->oom_score_adj, OOM_SCORE_ADJ_NO, NULL, updateOOMScoreAdj), + createEnumConfig("acl-pubsub-default", NULL, MODIFIABLE_CONFIG, acl_pubsub_default_enum, g_pserver->acl_pubusub_default, USER_FLAG_ALLCHANNELS, NULL, NULL), + createEnumConfig("sanitize-dump-payload", NULL, MODIFIABLE_CONFIG, sanitize_dump_payload_enum, cserver.sanitize_dump_payload, SANITIZE_DUMP_NO, NULL, NULL), /* Integer configs */ createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, cserver.dbnum, 16, INTEGER_CONFIG, NULL, NULL), - createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, g_pserver->port, 6379, INTEGER_CONFIG, NULL, NULL), /* TCP port. */ + createIntConfig("port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->port, 6379, INTEGER_CONFIG, NULL, updatePort), /* TCP port. */ createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->aof_rewrite_perc, 100, INTEGER_CONFIG, NULL, NULL), createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->cluster_slave_validity_factor, 10, INTEGER_CONFIG, NULL, NULL), /* Slave max data age factor. */ createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, g_pserver->list_max_ziplist_size, -2, INTEGER_CONFIG, NULL, NULL), @@ -2501,6 +2664,7 @@ standardConfig configs[] = { createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->slave_priority, 100, INTEGER_CONFIG, NULL, NULL), createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->repl_diskless_sync_delay, 5, INTEGER_CONFIG, NULL, NULL), createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->maxmemory_samples, 5, INTEGER_CONFIG, NULL, NULL), + createIntConfig("maxmemory-eviction-tenacity", NULL, MODIFIABLE_CONFIG, 0, 100, g_pserver->maxmemory_eviction_tenacity, 10, INTEGER_CONFIG, NULL, NULL), createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, cserver.maxidletime, 0, INTEGER_CONFIG, NULL, NULL), /* Default client timeout: infinite */ createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, g_pserver->slave_announce_port, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("tcp-backlog", NULL, IMMUTABLE_CONFIG, 0, INT_MAX, g_pserver->tcp_backlog, 511, INTEGER_CONFIG, NULL, NULL), /* TCP listen backlog. */ @@ -2509,8 +2673,8 @@ standardConfig configs[] = { createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->repl_timeout, 60, INTEGER_CONFIG, NULL, NULL), createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->repl_ping_slave_period, 10, INTEGER_CONFIG, NULL, NULL), createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL), - createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL), - createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->key_load_delay, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, g_pserver->rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, g_pserver->key_load_delay, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, cserver.active_expire_effort, 1, INTEGER_CONFIG, NULL, NULL), /* From 1 to 10. */ createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ), createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves), @@ -2522,7 +2686,6 @@ standardConfig configs[] = { createUIntConfig("loading-process-events-interval-keys", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->loading_process_events_interval_keys, 8192, MEMORY_CONFIG, NULL, NULL), /* Unsigned Long configs */ - createULongConfig("loading-process-events-interval-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->loading_process_events_interval_bytes, 2*1024*1024, MEMORY_CONFIG, NULL, NULL), createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, cserver.active_defrag_max_scan_fields, 1000, INTEGER_CONFIG, NULL, NULL), /* Default: keys with more than 1000 fields will be processed separately */ createULongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->slowlog_max_len, 128, INTEGER_CONFIG, NULL, NULL), createULongConfig("acllog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->acllog_max_len, 128, INTEGER_CONFIG, NULL, NULL), @@ -2549,28 +2712,36 @@ standardConfig configs[] = { createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->hll_sparse_max_bytes, 3000, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("tracking-table-max-keys", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 1 million keys max. */ + createSizeTConfig("client-query-buffer-limit", NULL, MODIFIABLE_CONFIG, 1024*1024, LONG_MAX, cserver.client_max_querybuf_len, 1024*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Default: 1GB max query buffer. */ /* Other configs */ createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->repl_backlog_time_limit, 60*60, INTEGER_CONFIG, NULL, NULL), /* Default: 1 hour */ createOffTConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->aof_rewrite_min_size, 64*1024*1024, MEMORY_CONFIG, NULL, NULL), + /* KeyDB Specific Configs */ + createULongConfig("loading-process-events-interval-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->loading_process_events_interval_bytes, 2*1024*1024, MEMORY_CONFIG, NULL, NULL), + createBoolConfig("multi-master-no-forward", NULL, MODIFIABLE_CONFIG, cserver.multimaster_no_forward, 0, validateMultiMasterNoForward, NULL), + createBoolConfig("allow-write-during-load", NULL, MODIFIABLE_CONFIG, g_pserver->fWriteDuringActiveLoad, 0, NULL, NULL), + #ifdef USE_OPENSSL - createIntConfig("tls-port", NULL, IMMUTABLE_CONFIG, 0, 65535, g_pserver->tls_port, 0, INTEGER_CONFIG, NULL, NULL), /* TCP port. */ - createIntConfig("tls-session-cache-size", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->tls_ctx_config.session_cache_size, 20*1024, INTEGER_CONFIG, NULL, updateTlsCfgInt), - createIntConfig("tls-session-cache-timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->tls_ctx_config.session_cache_timeout, 300, INTEGER_CONFIG, NULL, updateTlsCfgInt), - createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, g_pserver->tls_cluster, 0, NULL, NULL), - createBoolConfig("tls-replication", NULL, MODIFIABLE_CONFIG, g_pserver->tls_replication, 0, NULL, NULL), - createEnumConfig("tls-auth-clients", NULL, MODIFIABLE_CONFIG, tls_auth_clients_enum, g_pserver->tls_auth_clients, TLS_CLIENT_AUTH_YES, NULL, NULL), - createBoolConfig("tls-prefer-server-ciphers", NULL, MODIFIABLE_CONFIG, g_pserver->tls_ctx_config.prefer_server_ciphers, 0, NULL, updateTlsCfgBool), - createBoolConfig("tls-session-caching", NULL, MODIFIABLE_CONFIG, g_pserver->tls_ctx_config.session_caching, 1, NULL, updateTlsCfgBool), - createStringConfig("tls-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.cert_file, NULL, NULL, updateTlsCfg), - createStringConfig("tls-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.key_file, NULL, NULL, updateTlsCfg), - createStringConfig("tls-dh-params-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.dh_params_file, NULL, NULL, updateTlsCfg), - createStringConfig("tls-ca-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.ca_cert_file, NULL, NULL, updateTlsCfg), - createStringConfig("tls-ca-cert-dir", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.ca_cert_dir, NULL, NULL, updateTlsCfg), - createStringConfig("tls-protocols", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.protocols, NULL, NULL, updateTlsCfg), - createStringConfig("tls-ciphers", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.ciphers, NULL, NULL, updateTlsCfg), - createStringConfig("tls-ciphersuites", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.ciphersuites, NULL, NULL, updateTlsCfg), + createIntConfig("tls-port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG, NULL, updateTLSPort), /* TCP port. */ + createIntConfig("tls-session-cache-size", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tls_ctx_config.session_cache_size, 20*1024, INTEGER_CONFIG, NULL, updateTlsCfgInt), + createIntConfig("tls-session-cache-timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tls_ctx_config.session_cache_timeout, 300, INTEGER_CONFIG, NULL, updateTlsCfgInt), + createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, server.tls_cluster, 0, NULL, updateTlsCfgBool), + createBoolConfig("tls-replication", NULL, MODIFIABLE_CONFIG, server.tls_replication, 0, NULL, updateTlsCfgBool), + createEnumConfig("tls-auth-clients", NULL, MODIFIABLE_CONFIG, tls_auth_clients_enum, server.tls_auth_clients, TLS_CLIENT_AUTH_YES, NULL, NULL), + createBoolConfig("tls-prefer-server-ciphers", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.prefer_server_ciphers, 0, NULL, updateTlsCfgBool), + createBoolConfig("tls-session-caching", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.session_caching, 1, NULL, updateTlsCfgBool), + createStringConfig("tls-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.cert_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.key_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-client-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_cert_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-client-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_key_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-dh-params-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.dh_params_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ca-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ca-cert-dir", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_dir, NULL, NULL, updateTlsCfg), + createStringConfig("tls-protocols", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.protocols, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ciphers", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ciphers, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ciphersuites", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ciphersuites, NULL, NULL, updateTlsCfg), #endif /* NULL Terminator */ @@ -2590,12 +2761,17 @@ void configCommand(client *c) { if (c->argc == 2 && !strcasecmp(szFromObj(c->argv[1]),"help")) { const char *help[] = { -"GET -- Return parameters matching the glob-like and their values.", -"SET -- Set parameter to value.", -"RESETSTAT -- Reset statistics reported by INFO.", -"REWRITE -- Rewrite the configuration file.", +"GET ", +" Return parameters matching the glob-like and their values.", +"SET ", +" Set the configuration to .", +"RESETSTAT", +" Reset statistics reported by the INFO command.", +"REWRITE", +" Rewrite the configuration file.", NULL }; + addReplyHelp(c, help); } else if (!strcasecmp(szFromObj(c->argv[1]),"set") && c->argc == 4) { configSetCommand(c); @@ -2604,6 +2780,7 @@ NULL } else if (!strcasecmp(szFromObj(c->argv[1]),"resetstat") && c->argc == 2) { resetServerStats(); resetCommandTableStats(); + resetErrorTableStats(); addReply(c,shared.ok); } else if (!strcasecmp(szFromObj(c->argv[1]),"rewrite") && c->argc == 2) { if (cserver.configfile == NULL) { diff --git a/src/config.h b/src/config.h index 51fe6e253..b7e0a4f1e 100644 --- a/src/config.h +++ b/src/config.h @@ -35,10 +35,11 @@ #endif #ifdef __linux__ -#include #include #endif +#define CONFIG_DEFAULT_RDB_FILENAME "dump.rdb" + /* Define redis_fstat to fstat or fstat64() */ #if defined(__APPLE__) && !defined(MAC_OS_X_VERSION_10_6) #define redis_fstat fstat64 @@ -65,7 +66,7 @@ /* Test for backtrace() */ #if defined(__APPLE__) || (defined(__linux__) && defined(__GLIBC__)) || \ defined(__FreeBSD__) || ((defined(__OpenBSD__) || defined(__NetBSD__)) && defined(USE_BACKTRACE))\ - || defined(__DragonFly__) + || defined(__DragonFly__) || (defined(__UCLIBC__) && defined(__UCLIBC_HAS_BACKTRACE__)) #define HAVE_BACKTRACE 1 #endif @@ -87,6 +88,7 @@ #include #ifdef _DTRACE_VERSION #define HAVE_EVPORT 1 +#define HAVE_PSINFO 1 #endif #endif @@ -97,21 +99,23 @@ #define redis_fsync fsync #endif -/* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use - * the plain fsync() call. */ -#ifdef __linux__ -#if defined(__GLIBC__) && defined(__GLIBC_PREREQ) -#if (LINUX_VERSION_CODE >= 0x020611 && __GLIBC_PREREQ(2, 6)) -#define HAVE_SYNC_FILE_RANGE 1 -#endif +#if __GNUC__ >= 4 +#define redis_unreachable __builtin_unreachable #else -#if (LINUX_VERSION_CODE >= 0x020611) -#define HAVE_SYNC_FILE_RANGE 1 -#endif -#endif +#define redis_unreachable abort #endif -#ifdef HAVE_SYNC_FILE_RANGE +#if __GNUC__ >= 3 +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define likely(x) (x) +#define unlikely(x) (x) +#endif + +/* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use + * the plain fsync() call. */ +#if (defined(__linux__) && defined(SYNC_FILE_RANGE_WAIT_BEFORE)) #define rdb_fsync_range(fd,off,size) sync_file_range(fd,off,size,SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE) #else #define rdb_fsync_range(fd,off,size) fsync(fd) @@ -128,7 +132,7 @@ #define ESOCKTNOSUPPORT 0 #endif -#if ((defined __linux && defined(__GLIBC__)) || defined __APPLE__) +#if (defined __linux || defined __APPLE__) #define USE_SETPROCTITLE #define INIT_SETPROCTITLE_REPLACEMENT #ifdef __cplusplus @@ -247,11 +251,11 @@ void setproctitle(const char *fmt, ...); #elif defined __NetBSD__ #include #define redis_set_thread_title(name) pthread_setname_np(pthread_self(), "%s", name) +#elif defined __HAIKU__ +#include +#define redis_set_thread_title(name) rename_thread(find_thread(0), name) #else #if (defined __APPLE__ && defined(MAC_OS_X_VERSION_10_7)) -#ifdef __cplusplus -extern "C" -#endif int pthread_setname_np(const char *name); #include #define redis_set_thread_title(name) pthread_setname_np(name) diff --git a/src/connection.cpp b/src/connection.cpp index 8ba75264e..11d1e0809 100644 --- a/src/connection.cpp +++ b/src/connection.cpp @@ -147,8 +147,7 @@ void *connGetPrivateData(connection *conn) { /* Close the connection and free resources. */ static void connSocketClose(connection *conn) { if (conn->fd != -1) { - aeDeleteFileEvent(serverTL->el,conn->fd,AE_READABLE); - aeDeleteFileEvent(serverTL->el,conn->fd,AE_WRITABLE); + aeDeleteFileEvent(serverTL->el,conn->fd,AE_READABLE | AE_WRITABLE); close(conn->fd); conn->fd = -1; } @@ -384,15 +383,15 @@ int connGetSocketError(connection *conn) { } int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port) { - return anetPeerToString(conn ? conn->fd : -1, ip, ip_len, port); -} - -int connFormatPeer(connection *conn, char *buf, size_t buf_len) { - return anetFormatPeer(conn ? conn->fd : -1, buf, buf_len); + return anetFdToString(conn ? conn->fd : -1, ip, ip_len, port, FD_TO_PEER_NAME); } int connSockName(connection *conn, char *ip, size_t ip_len, int *port) { - return anetSockName(conn->fd, ip, ip_len, port); + return anetFdToString(conn->fd, ip, ip_len, port, FD_TO_SOCK_NAME); +} + +int connFormatFdAddr(connection *conn, char *buf, size_t buf_len, int fd_to_str_type) { + return anetFormatFdAddr(conn ? conn->fd : -1, buf, buf_len, fd_to_str_type); } int connBlock(connection *conn) { @@ -447,7 +446,7 @@ void connSetThreadAffinity(connection *conn, int cpu) { * For sockets, we always return "fd=" to maintain compatibility. */ const char *connGetInfo(connection *conn, char *buf, size_t buf_len) { - snprintf(buf, buf_len-1, "fd=%i", conn->fd); + snprintf(buf, buf_len-1, "fd=%i", conn == NULL ? -1 : conn->fd); return buf; } diff --git a/src/connection.h b/src/connection.h index 606137229..bffde3f5c 100644 --- a/src/connection.h +++ b/src/connection.h @@ -236,7 +236,7 @@ int connKeepAlive(connection *conn, int interval); int connSendTimeout(connection *conn, long long ms); int connRecvTimeout(connection *conn, long long ms); int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port); -int connFormatPeer(connection *conn, char *buf, size_t buf_len); +int connFormatFdAddr(connection *conn, char *buf, size_t buf_len, int fd_to_str_type); int connSockName(connection *conn, char *ip, size_t ip_len, int *port); const char *connGetInfo(connection *conn, char *buf, size_t buf_len); diff --git a/src/crc64.c b/src/crc64.c index 4cbc019f6..6c9432c4a 100644 --- a/src/crc64.c +++ b/src/crc64.c @@ -134,7 +134,7 @@ int crc64Test(int argc, char *argv[]) { printf("[calcula]: e9c6d914c4b8d9ca == %016" PRIx64 "\n", (uint64_t)_crc64(0, "123456789", 9)); printf("[64speed]: e9c6d914c4b8d9ca == %016" PRIx64 "\n", - (uint64_t)crc64(0, "123456789", 9)); + (uint64_t)crc64(0, (unsigned char*)"123456789", 9)); char li[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed " "do eiusmod tempor incididunt ut labore et dolore magna " "aliqua. Ut enim ad minim veniam, quis nostrud exercitation " @@ -146,7 +146,7 @@ int crc64Test(int argc, char *argv[]) { printf("[calcula]: c7794709e69683b3 == %016" PRIx64 "\n", (uint64_t)_crc64(0, li, sizeof(li))); printf("[64speed]: c7794709e69683b3 == %016" PRIx64 "\n", - (uint64_t)crc64(0, li, sizeof(li))); + (uint64_t)crc64(0, (unsigned char*)li, sizeof(li))); return 0; } diff --git a/src/crcspeed.c b/src/crcspeed.c index 81a80ce8e..67cb8fd9f 100644 --- a/src/crcspeed.c +++ b/src/crcspeed.c @@ -85,7 +85,7 @@ static inline uint64_t rev8(uint64_t a) { /* This function is called once to initialize the CRC table for use on a big-endian architecture. */ void crcspeed64big_init(crcfn64 fn, uint64_t big_table[8][256]) { - /* Create the little endian table then reverse all the entires. */ + /* Create the little endian table then reverse all the entries. */ crcspeed64little_init(fn, big_table); for (int k = 0; k < 8; k++) { for (int n = 0; n < 256; n++) { @@ -95,7 +95,7 @@ void crcspeed64big_init(crcfn64 fn, uint64_t big_table[8][256]) { } void crcspeed16big_init(crcfn16 fn, uint16_t big_table[8][256]) { - /* Create the little endian table then reverse all the entires. */ + /* Create the little endian table then reverse all the entries. */ crcspeed16little_init(fn, big_table); for (int k = 0; k < 8; k++) { for (int n = 0; n < 256; n++) { diff --git a/src/cron.cpp b/src/cron.cpp index 8f2592c73..5a7a853be 100644 --- a/src/cron.cpp +++ b/src/cron.cpp @@ -82,7 +82,7 @@ void executeCronJobExpireHook(const char *key, robj *o) client *cFake = createClient(nullptr, IDX_EVENT_LOOP_MAIN); cFake->lock.lock(); cFake->authenticated = 1; - cFake->puser = nullptr; + cFake->user = nullptr; selectDb(cFake, job->dbNum); serverAssert(cFake->argc == 0); diff --git a/src/db.cpp b/src/db.cpp index fa32e78c0..498aff0aa 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -48,7 +48,7 @@ struct dbBackup { int keyIsExpired(redisDb *db, robj *key); int expireIfNeeded(redisDb *db, robj *key, robj *o); -void dbOverwriteCore(redisDb *db, dictEntry *de, robj *key, robj *val, bool fUpdateMvcc, bool fRemoveExpire); +void dbOverwriteCore(redisDb *db, dictEntry *de, sds key, robj *val, bool fUpdateMvcc, bool fRemoveExpire); /* Update LFU when an object is accessed. * Firstly, decrement the counter if the decrement time is reached. @@ -135,9 +135,8 @@ robj_roptr lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) { serverAssert(GlobalLocksAcquired()); if (expireIfNeeded(db,key) == 1) { - /* Key expired. If we are in the context of a master, expireIfNeeded() - * returns 0 only when the key does not exist at all, so it's safe - * to return NULL ASAP. */ + /* If we are in the context of a master, expireIfNeeded() returns 1 + * when the key is no longer valid, so we can return NULL ASAP. */ if (listLength(g_pserver->masters) == 0) goto keymiss; @@ -208,23 +207,19 @@ robj *lookupKeyWriteOrReply(client *c, robj *key, robj *reply) { return o; } -int dbAddCore(redisDb *db, robj *key, robj *val, bool fUpdateMvcc) { +int dbAddCore(redisDb *db, sds key, robj *val, bool fUpdateMvcc) { serverAssert(!val->FExpires()); - sds copy = sdsdup(szFromObj(key)); + sds copy = sdsdup(key); int retval = dictAdd(db->dict, copy, val); uint64_t mvcc = getMvccTstamp(); if (fUpdateMvcc) { - setMvccTstamp(key, mvcc); setMvccTstamp(val, mvcc); } if (retval == DICT_OK) { - if (val->type == OBJ_LIST || - val->type == OBJ_ZSET || - val->type == OBJ_STREAM) - signalKeyAsReady(db, key); - if (g_pserver->cluster_enabled) slotToKeyAdd(szFromObj(key)); + signalKeyAsReady(db, key, val->type); + if (g_pserver->cluster_enabled) slotToKeyAdd(key); } else { @@ -240,14 +235,17 @@ int dbAddCore(redisDb *db, robj *key, robj *val, bool fUpdateMvcc) { * The program is aborted if the key already exists. */ void dbAdd(redisDb *db, robj *key, robj *val) { - int retval = dbAddCore(db, key, val, true /* fUpdateMvcc */); + int retval = dbAddCore(db, szFromObj(key), val, true /* fUpdateMvcc */); serverAssertWithInfo(NULL,key,retval == DICT_OK); } -void dbOverwriteCore(redisDb *db, dictEntry *de, robj *key, robj *val, bool fUpdateMvcc, bool fRemoveExpire) +void dbOverwriteCore(redisDb *db, dictEntry *de, sds keySds, robj *val, bool fUpdateMvcc, bool fRemoveExpire) { dictEntry auxentry = *de; robj *old = (robj*)dictGetVal(de); + redisObjectStack keyO; + initStaticStringObject(keyO, keySds); + robj *key = &keyO; if (old->FExpires()) { if (fRemoveExpire) { @@ -269,10 +267,14 @@ void dbOverwriteCore(redisDb *db, dictEntry *de, robj *key, robj *val, bool fUpd setMvccTstamp(val, getMvccTstamp()); } + /* Although the key is not really deleted from the database, we regard + overwrite as two steps of unlink+add, so we still need to call the unlink + callback of the module. */ + moduleNotifyKeyUnlink(key,old); dictSetVal(db->dict, de, val); if (g_pserver->lazyfree_lazy_server_del) { - freeObjAsync(old); + freeObjAsync(key,old); dictSetVal(db->dict, &auxentry, NULL); } @@ -288,15 +290,15 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) { dictEntry *de = dictFind(db->dict,ptrFromObj(key)); serverAssertWithInfo(NULL,key,de != NULL); - dbOverwriteCore(db, de, key, val, !!g_pserver->fActiveReplica, false); + dbOverwriteCore(db, de, szFromObj(key), val, !!g_pserver->fActiveReplica, false); } /* Insert a key, handling duplicate keys according to fReplace */ -int dbMerge(redisDb *db, robj *key, robj *val, int fReplace) +int dbMerge(redisDb *db, sds key, robj *val, int fReplace) { if (fReplace) { - dictEntry *de = dictFind(db->dict, ptrFromObj(key)); + dictEntry *de = dictFind(db->dict, key); if (de == nullptr) return (dbAddCore(db, key, val, false /* fUpdateMvcc */) == DICT_OK); @@ -332,7 +334,7 @@ void genericSetKey(client *c, redisDb *db, robj *key, robj *val, int keepttl, in dbAdd(db,key,val); } else { updateDbValAccess(de, LOOKUP_NONE); - dbOverwriteCore(db,de,key,val,!!g_pserver->fActiveReplica,!keepttl); + dbOverwriteCore(db,de,szFromObj(key),val,!!g_pserver->fActiveReplica,!keepttl); } incrRefCount(val); if (signal) signalModifiedKey(c,db,key); @@ -343,12 +345,6 @@ void setKey(client *c, redisDb *db, robj *key, robj *val) { genericSetKey(c,db,key,val,0,1); } -/* Return true if the specified key exists in the specified database. - * LRU/LFU info is not updated in any way. */ -int dbExists(redisDb *db, robj *key) { - return dictFind(db->dict,ptrFromObj(key)) != NULL; -} - /* Return a random key, in form of a Redis object. * If there are no keys, NULL is returned. * @@ -400,10 +396,14 @@ int dbSyncDelete(redisDb *db, robj *key) { /* Deleting an entry from the expires dict will not free the sds of * the key, because it is shared with the main dictionary. */ - dictEntry *de = dictFind(db->dict, szFromObj(key)); + dictEntry *de = dictUnlink(db->dict,ptrFromObj(key)); if (de != nullptr && ((robj*)dictGetVal(de))->FExpires()) removeExpireCore(db, key, de); - if (dictDelete(db->dict,ptrFromObj(key)) == DICT_OK) { + if (de) { + robj *val = (robj*)dictGetVal(de); + /* Tells the module that the key has been unlinked from the database. */ + moduleNotifyKeyUnlink(key,val); + dictFreeUnlinkedEntry(db->dict,de); if (g_pserver->cluster_enabled) slotToKeyDel(szFromObj(key)); return 1; } else { @@ -485,7 +485,6 @@ long long emptyDbStructure(redisDb *dbarray, int dbnum, int async, } /* Because all keys of database are removed, reset average ttl. */ dbarray[j].avg_ttl = 0; - dbarray[j].last_expire_set = 0; } return removed; @@ -523,7 +522,7 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { /* Make sure the WATCHed keys are affected by the FLUSH* commands. * Note that we need to call the function while the keys are still * there. */ - signalFlushedDb(dbnum); + signalFlushedDb(dbnum, async); /* Empty redis database structure. */ removed = emptyDbStructure(g_pserver->db, dbnum, async, callback); @@ -568,6 +567,10 @@ dbBackup *backupDb(void) { sizeof(g_pserver->cluster->slots_keys_count)); } + moduleFireServerEvent(REDISMODULE_EVENT_REPL_BACKUP, + REDISMODULE_SUBEVENT_REPL_BACKUP_CREATE, + NULL); + return backup; } @@ -589,6 +592,10 @@ void discardDbBackup(dbBackup *buckup, int flags, void(callback)(void*)) { /* Release buckup. */ zfree(buckup->dbarray); zfree(buckup); + + moduleFireServerEvent(REDISMODULE_EVENT_REPL_BACKUP, + REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD, + NULL); } /* Restore the previously created backup (discarding what currently resides @@ -617,6 +624,10 @@ void restoreDbBackup(dbBackup *buckup) { /* Release buckup. */ zfree(buckup->dbarray); zfree(buckup); + + moduleFireServerEvent(REDISMODULE_EVENT_REPL_BACKUP, + REDISMODULE_SUBEVENT_REPL_BACKUP_RESTORE, + NULL); } int selectDb(client *c, int id) { @@ -651,7 +662,7 @@ void signalModifiedKey(client *c, redisDb *db, robj *key) { trackingInvalidateKey(c,key); } -void signalFlushedDb(int dbid) { +void signalFlushedDb(int dbid, int async) { int startdb, enddb; if (dbid == -1) { startdb = 0; @@ -664,7 +675,7 @@ void signalFlushedDb(int dbid) { touchAllWatchedKeysInDb(&g_pserver->db[j], NULL); } - trackingInvalidateKeysOnFlush(dbid); + trackingInvalidateKeysOnFlush(async); } /*----------------------------------------------------------------------------- @@ -674,21 +685,23 @@ void signalFlushedDb(int dbid) { /* Return the set of flags to use for the emptyDb() call for FLUSHALL * and FLUSHDB commands. * - * Currently the command just attempts to parse the "ASYNC" option. It - * also checks if the command arity is wrong. + * sync: flushes the database in an sync manner. + * async: flushes the database in an async manner. + * no option: determine sync or async according to the value of lazyfree-lazy-user-flush. * * On success C_OK is returned and the flags are stored in *flags, otherwise * C_ERR is returned and the function sends an error to the client. */ int getFlushCommandFlags(client *c, int *flags) { /* Parse the optional ASYNC option. */ - if (c->argc > 1) { - if (c->argc > 2 || strcasecmp(szFromObj(c->argv[1]),"async")) { - addReply(c,shared.syntaxerr); - return C_ERR; - } - *flags = EMPTYDB_ASYNC; - } else { + if (c->argc == 2 && !strcasecmp(szFromObj(c->argv[1]),"sync")) { *flags = EMPTYDB_NO_FLAGS; + } else if (c->argc == 2 && !strcasecmp(szFromObj(c->argv[1]),"async")) { + *flags = EMPTYDB_ASYNC; + } else if (c->argc == 1) { + *flags = g_pserver->lazyfree_lazy_user_flush ? EMPTYDB_ASYNC : EMPTYDB_NO_FLAGS; + } else { + addReplyErrorObject(c,shared.syntaxerr); + return C_ERR; } return C_OK; } @@ -696,7 +709,7 @@ int getFlushCommandFlags(client *c, int *flags) { /* Flushes the whole server data set. */ void flushAllDataAndResetRDB(int flags) { g_pserver->dirty += emptyDb(-1,flags,NULL); - if (g_pserver->rdb_child_pid != -1) killRDBChild(); + if (g_pserver->child_type == CHILD_TYPE_RDB) killRDBChild(); if (g_pserver->saveparamslen > 0) { /* Normally rdbSave() will reset dirty, but we don't want this here * as otherwise FLUSHALL will not be replicated nor put into the AOF. */ @@ -706,6 +719,9 @@ void flushAllDataAndResetRDB(int flags) { rdbSave(rsiptr); g_pserver->dirty = saved_dirty; } + + /* Without that extra dirty++, when db was already empty, FLUSHALL will + * not be replicated nor put into the AOF. */ g_pserver->dirty++; #if defined(USE_JEMALLOC) /* jemalloc 5 doesn't release pages back to the OS when there's no traffic. @@ -791,10 +807,9 @@ void mexistsCommand(client *c) { } void selectCommand(client *c) { - long id; + int id; - if (getLongFromObjectOrReply(c, c->argv[1], &id, - "invalid DB index") != C_OK) + if (getIntFromObjectOrReply(c, c->argv[1], &id, NULL) != C_OK) return; if (g_pserver->cluster_enabled && id != 0) { @@ -937,7 +952,7 @@ void scanGenericCommand(client *c, robj_roptr o, unsigned long cursor) { } if (count < 1) { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); goto cleanup; } @@ -956,7 +971,7 @@ void scanGenericCommand(client *c, robj_roptr o, unsigned long cursor) { type = szFromObj(c->argv[i+1]); i+= 2; } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); goto cleanup; } } @@ -1035,7 +1050,7 @@ void scanGenericCommand(client *c, robj_roptr o, unsigned long cursor) { int filter = 0; /* Filter element if it does not match the pattern. */ - if (!filter && use_pattern) { + if (use_pattern) { if (sdsEncodedObject(kobj)) { if (!stringmatchlen(pat, patlen, szFromObj(kobj), sdslen(szFromObj(kobj)), 0)) filter = 1; @@ -1070,6 +1085,7 @@ void scanGenericCommand(client *c, robj_roptr o, unsigned long cursor) { * value, or skip it if it was not filtered: we only match keys. */ if (o && (o->type == OBJ_ZSET || o->type == OBJ_HASH)) { node = nextnode; + serverAssert(node); /* assertion for valgrind (avoid NPD) */ nextnode = listNextNode(node); if (filter) { kobj = (robj*)listNodeValue(node); @@ -1143,7 +1159,7 @@ void shutdownCommand(client *c) { int flags = 0; if (c->argc > 2) { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } else if (c->argc == 2) { if (!strcasecmp(szFromObj(c->argv[1]),"nosave")) { @@ -1151,7 +1167,7 @@ void shutdownCommand(client *c) { } else if (!strcasecmp(szFromObj(c->argv[1]),"save")) { flags |= SHUTDOWN_SAVE; } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } @@ -1220,8 +1236,7 @@ void renamenxCommand(client *c) { void moveCommand(client *c) { robj *o; redisDb *src, *dst; - int srcid; - long long dbid; + int srcid, dbid; if (g_pserver->cluster_enabled) { addReplyError(c,"MOVE is not allowed in cluster mode"); @@ -1232,11 +1247,11 @@ void moveCommand(client *c) { src = c->db; srcid = c->db->id; - if (getLongLongFromObject(c->argv[2],&dbid) == C_ERR || - dbid < INT_MIN || dbid > INT_MAX || - selectDb(c,dbid) == C_ERR) - { - addReply(c,shared.outofrangeerr); + if (getIntFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK) + return; + + if (selectDb(c,dbid) == C_ERR) { + addReplyError(c,"DB index is out of range"); return; } dst = c->db; @@ -1245,7 +1260,7 @@ void moveCommand(client *c) { /* If the user is moving using as target the same * DB as the source DB it is probably an error. */ if (src == dst) { - addReply(c,shared.sameobjecterr); + addReplyErrorObject(c,shared.sameobjecterr); return; } @@ -1288,6 +1303,110 @@ void moveCommand(client *c) { addReply(c,shared.cone); } +void copyCommand(client *c) { + robj *o; + redisDb *src, *dst; + int srcid, dbid; + expireEntry *expire = nullptr; + int j, replace = 0, fdelete = 0; + + /* Obtain source and target DB pointers + * Default target DB is the same as the source DB + * Parse the REPLACE option and targetDB option. */ + src = c->db; + dst = c->db; + srcid = c->db->id; + dbid = c->db->id; + for (j = 3; j < c->argc; j++) { + int additional = c->argc - j - 1; + if (!strcasecmp(szFromObj(c->argv[j]),"replace")) { + replace = 1; + } else if (!strcasecmp(szFromObj(c->argv[j]), "db") && additional >= 1) { + if (getIntFromObjectOrReply(c, c->argv[j+1], &dbid, NULL) != C_OK) + return; + + if (selectDb(c, dbid) == C_ERR) { + addReplyError(c,"DB index is out of range"); + return; + } + dst = c->db; + selectDb(c,srcid); /* Back to the source DB */ + j++; /* Consume additional arg. */ + } else { + addReplyErrorObject(c,shared.syntaxerr); + return; + } + } + + if ((g_pserver->cluster_enabled == 1) && (srcid != 0 || dbid != 0)) { + addReplyError(c,"Copying to another database is not allowed in cluster mode"); + return; + } + + /* If the user select the same DB as + * the source DB and using newkey as the same key + * it is probably an error. */ + robj *key = c->argv[1]; + robj *newkey = c->argv[2]; + if (src == dst && (sdscmp(szFromObj(key), szFromObj(newkey)) == 0)) { + addReplyErrorObject(c,shared.sameobjecterr); + return; + } + + /* Check if the element exists and get a reference */ + o = lookupKeyWrite(c->db, key); + if (!o) { + addReply(c,shared.czero); + return; + } + expire = getExpire(c->db,key); + + /* Return zero if the key already exists in the target DB. + * If REPLACE option is selected, delete newkey from targetDB. */ + if (lookupKeyWrite(dst,newkey) != NULL) { + if (replace) { + fdelete = 1; + } else { + addReply(c,shared.czero); + return; + } + } + + /* Duplicate object according to object's type. */ + robj *newobj; + switch(o->type) { + case OBJ_STRING: newobj = dupStringObject(o); break; + case OBJ_LIST: newobj = listTypeDup(o); break; + case OBJ_SET: newobj = setTypeDup(o); break; + case OBJ_ZSET: newobj = zsetDup(o); break; + case OBJ_HASH: newobj = hashTypeDup(o); break; + case OBJ_STREAM: newobj = streamDup(o); break; + case OBJ_MODULE: + newobj = moduleTypeDupOrReply(c, key, newkey, o); + if (!newobj) return; + break; + default: + addReplyError(c, "unknown type object"); + return; + } + + if (fdelete) { + dbDelete(dst,newkey); + } + + dbAdd(dst,newkey,newobj); + if (expire != nullptr) { + if (expire != nullptr) setExpire(c, dst, newkey, expire->duplicate()); + } + + /* OK! key copied */ + signalModifiedKey(c,dst,c->argv[2]); + notifyKeyspaceEvent(NOTIFY_GENERIC,"copy_to",c->argv[2],dst->id); + + g_pserver->dirty++; + addReply(c,shared.cone); +} + /* Helper function for dbSwapDatabases(): scans the list of keys that have * one or more blocked clients for B[LR]POP or other blocking commands * and signal the keys as ready if they are of the right type. See the comment @@ -1298,10 +1417,7 @@ void scanDatabaseForReadyLists(redisDb *db) { while((de = dictNext(di)) != NULL) { robj *key = (robj*)dictGetKey(de); robj *value = lookupKey(db,key,LOOKUP_NOTOUCH); - if (value && (value->type == OBJ_LIST || - value->type == OBJ_STREAM || - value->type == OBJ_ZSET)) - signalKeyAsReady(db, key); + if (value) signalKeyAsReady(db, key, value->type); } dictReleaseIterator(di); } @@ -1314,7 +1430,7 @@ void scanDatabaseForReadyLists(redisDb *db) { * * Returns C_ERR if at least one of the DB ids are out of range, otherwise * C_OK is returned. */ -int dbSwapDatabases(long id1, long id2) { +int dbSwapDatabases(int id1, int id2) { if (id1 < 0 || id1 >= cserver.dbnum || id2 < 0 || id2 >= cserver.dbnum) return C_ERR; if (id1 == id2) return C_OK; @@ -1350,7 +1466,7 @@ int dbSwapDatabases(long id1, long id2) { /* SWAPDB db1 db2 */ void swapdbCommand(client *c) { - long id1, id2; + int id1, id2; /* Not allowed in cluster mode: we have just DB 0 there. */ if (g_pserver->cluster_enabled) { @@ -1359,11 +1475,11 @@ void swapdbCommand(client *c) { } /* Get the two DBs indexes. */ - if (getLongFromObjectOrReply(c, c->argv[1], &id1, + if (getIntFromObjectOrReply(c, c->argv[1], &id1, "invalid first DB index") != C_OK) return; - if (getLongFromObjectOrReply(c, c->argv[2], &id2, + if (getIntFromObjectOrReply(c, c->argv[2], &id2, "invalid second DB index") != C_OK) return; @@ -1565,11 +1681,8 @@ void propagateExpire(redisDb *db, robj *key, int lazy) { incrRefCount(argv[0]); incrRefCount(argv[1]); - if (g_pserver->aof_state != AOF_OFF) - feedAppendOnlyFile(cserver.delCommand,db->id,argv,2); - // Active replicas do their own expiries, do not propogate - if (!g_pserver->fActiveReplica) - replicationFeedSlaves(g_pserver->slaves,db->id,argv,2); + if (!g_pserver->fActiveReplica) // Active replicas do their own expiries, do not propogate + propagate(cserver.delCommand,db->id,argv,2,PROPAGATE_AOF|PROPAGATE_REPL); decrRefCount(argv[0]); decrRefCount(argv[1]); @@ -1699,6 +1812,12 @@ int expireIfNeeded(redisDb *db, robj *key) { * we think the key is expired at this time. */ if (listLength(g_pserver->masters) && !g_pserver->fActiveReplica) return 1; + /* If clients are paused, we keep the current dataset constant, + * but return to the client what we believe is the right state. Typically, + * at the end of the pause we will properly expire the key OR we will + * have failed over and the new primary will send us the expire. */ + if (checkClientPauseTimeoutAndReturnIfPaused()) return 1; + /* Delete the key */ g_pserver->stat_expiredkeys++; propagateExpire(db,key,g_pserver->lazyfree_lazy_expire); @@ -1812,58 +1931,53 @@ void getKeysFreeResult(getKeysResult *result) { } /* Helper function to extract keys from following commands: + * COMMAND [destkey] [...] [...] ... + * + * eg: + * ZUNION ... * ZUNIONSTORE ... - * ZINTERSTORE ... */ -int zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) { + * + * 'storeKeyOfs': destkey index, 0 means destkey not exists. + * 'keyCountOfs': num-keys index. + * 'firstKeyOfs': firstkey index. + * 'keyStep': the interval of each key, usually this value is 1. + * */ +int genericGetKeys(int storeKeyOfs, int keyCountOfs, int firstKeyOfs, int keyStep, + robj **argv, int argc, getKeysResult *result) { int i, num, *keys; - UNUSED(cmd); - num = atoi(szFromObj(argv[2])); + num = atoi(szFromObj(argv[keyCountOfs])); /* Sanity check. Don't return any key if the command is going to - * reply with syntax error. */ - if (num < 1 || num > (argc-3)) { + * reply with syntax error. (no input keys). */ + if (num < 1 || num > (argc - firstKeyOfs)/keyStep) { result->numkeys = 0; return 0; } - /* Keys in z{union,inter}store come from two places: - * argv[1] = storage key, - * argv[3...n] = keys to intersect */ - /* Total keys = {union,inter} keys + storage key */ - keys = getKeysPrepareResult(result, num+1); - result->numkeys = num+1; + int numkeys = storeKeyOfs ? num + 1 : num; + keys = getKeysPrepareResult(result, numkeys); + result->numkeys = numkeys; - /* Add all key positions for argv[3...n] to keys[] */ - for (i = 0; i < num; i++) keys[i] = 3+i; - - /* Finally add the argv[1] key position (the storage key target). */ - keys[num] = 1; + /* Add all key positions for argv[firstKeyOfs...n] to keys[] */ + for (i = 0; i < num; i++) keys[i] = firstKeyOfs+(i*keyStep); + if (storeKeyOfs) keys[num] = storeKeyOfs; return result->numkeys; } -/* Helper function to extract keys from the following commands: - * EVAL