diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml new file mode 100644 index 000000000..ee9ac1bbf --- /dev/null +++ b/.github/workflows/daily.yml @@ -0,0 +1,308 @@ +name: Daily + +on: + pull_request: + branches: + # any PR to a release branch. + - '[0-9].[0-9]' + schedule: + - cron: '0 0 * * *' + +jobs: + + test-ubuntu-jemalloc: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: make REDIS_CFLAGS='-Werror -DREDIS_TEST' + - name: test + run: | + sudo apt-get install tcl8.6 + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + - name: unittest + run: ./src/redis-server test all + + test-ubuntu-libc-malloc: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: make MALLOC=libc + - name: test + run: | + sudo apt-get install tcl8.6 + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + + test-ubuntu-no-malloc-usable-size: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: make MALLOC=libc CFLAGS=-DNO_MALLOC_USABLE_SIZE + - name: test + run: | + sudo apt-get install tcl8.6 + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + + test-ubuntu-32bit: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: | + sudo apt-get update && sudo apt-get install libc6-dev-i386 + make 32bit REDIS_CFLAGS='-Werror -DREDIS_TEST' + - name: test + run: | + sudo apt-get install tcl8.6 + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: | + make -C tests/modules 32bit # the script below doesn't have an argument, we must build manually ahead of time + ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + - name: unittest + run: ./src/redis-server test all + + test-ubuntu-tls: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: | + make BUILD_TLS=yes + - name: test + run: | + sudo apt-get install tcl8.6 tcl-tls + ./utils/gen-test-certs.sh + ./runtest --accurate --verbose --tls --dump-logs + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: | + ./runtest-moduleapi --verbose --tls + ./runtest-moduleapi --verbose + - name: sentinel tests + run: | + ./runtest-sentinel --tls + ./runtest-sentinel + - name: cluster tests + run: | + ./runtest-cluster --tls + ./runtest-cluster + + test-ubuntu-io-threads: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: | + make + - name: test + run: | + sudo apt-get install tcl8.6 tcl-tls + ./runtest --config io-threads 4 --config io-threads-do-reads yes --accurate --verbose --tags network --dump-logs + - name: cluster tests + run: | + ./runtest-cluster --config io-threads 4 --config io-threads-do-reads yes + + test-valgrind: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: make valgrind REDIS_CFLAGS='-Werror -DREDIS_TEST' + - name: test + run: | + sudo apt-get update + sudo apt-get install tcl8.6 valgrind -y + ./runtest --valgrind --verbose --clients 1 --dump-logs + - name: module api test + run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 + - name: unittest + run: | + valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/redis-server test all + if grep -q 0x err.txt; then cat err.txt; exit 1; fi + + test-valgrind-no-malloc-usable-size: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: make valgrind CFLAGS="-DNO_MALLOC_USABLE_SIZE" + - name: test + run: | + sudo apt-get update + sudo apt-get install tcl8.6 valgrind -y + ./runtest --valgrind --verbose --clients 1 --dump-logs + - name: module api test + run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 + + test-centos7-jemalloc: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + container: centos:7 + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: | + yum -y install gcc make + make + - name: test + run: | + yum -y install which tcl + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + + test-centos7-tls: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + container: centos:7 + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: | + yum -y install centos-release-scl epel-release + yum -y install devtoolset-7 openssl-devel openssl + scl enable devtoolset-7 "make BUILD_TLS=yes" + - name: test + run: | + yum -y install tcl tcltls + ./utils/gen-test-certs.sh + ./runtest --accurate --verbose --tls --dump-logs + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: | + ./runtest-moduleapi --verbose --tls + ./runtest-moduleapi --verbose + - name: sentinel tests + run: | + ./runtest-sentinel --tls + ./runtest-sentinel + - name: cluster tests + run: | + ./runtest-cluster --tls + ./runtest-cluster + + test-macos-latest: + runs-on: macos-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: make + - name: test + run: | + ./runtest --accurate --verbose --no-latency --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + + test-freebsd: + runs-on: macos-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: test + uses: vmactions/freebsd-vm@v0.1.2 + with: + usesh: true + sync: rsync + prepare: pkg install -y bash gmake lang/tcl86 + run: > + gmake && + ./runtest --accurate --verbose --no-latency --dump-logs && + MAKE=gmake ./runtest-moduleapi --verbose && + ./runtest-sentinel && + ./runtest-cluster + + test-alpine-jemalloc: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + container: alpine:latest + steps: + - uses: actions/checkout@v2 + - name: make + run: | + apk add build-base + make REDIS_CFLAGS='-Werror' + - name: test + run: | + apk add tcl procps + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + + test-alpine-libc-malloc: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + container: alpine:latest + steps: + - uses: actions/checkout@v2 + - name: make + run: | + apk add build-base + make REDIS_CFLAGS='-Werror' USE_JEMALLOC=no CFLAGS=-DUSE_MALLOC_USABLE_SIZE + - name: test + run: | + apk add tcl procps + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster diff --git a/00-RELEASENOTES b/00-RELEASENOTES new file mode 100644 index 000000000..8a1405e41 --- /dev/null +++ b/00-RELEASENOTES @@ -0,0 +1,532 @@ +Redis 6.2 release notes +======================= + +-------------------------------------------------------------------------------- +Upgrade urgency levels: + +LOW: No need to upgrade unless there are new features you want to use. +MODERATE: Program an upgrade of the server, but it's not urgent. +HIGH: There is a critical bug that may affect a subset of users. Upgrade! +CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. +SECURITY: There are security fixes in the release. +-------------------------------------------------------------------------------- + +================================================================================ +Redis 6.2.2 Released Mon April 19 19:00:00 IST 2021 +================================================================================ + +Upgrade urgency: HIGH, if you're using ACL and pub/sub, CONFIG REWRITE, or +suffering from performance regression. see below. + +Bug fixes for regressions in previous releases of Redis 6.2: +* Fix BGSAVE, AOFRW, and replication slowdown due to child reporting CoW (#8645) +* Fix short busy loop when timer event is about to fire (#8764) +* Fix default user, overwritten and reset users losing pubsub channel permissions (#8723) +* Fix config rewrite with an empty `save` config resulsing in default `save` values (#8719) +* Fix not starting on alpine/libmusl without IPv6 (#8655) +* Fix issues with propagation and MULTI/EXEC in modules (#8617) + Several issues around nested calls and thread safe contexts + +Bug fixes that are only applicable to previous releases of Redis 6.2: +* ACL Pub/Sub channels permission handling for save/load scenario (#8794) +* Fix early rejection of PUBLISH inside MULTI-EXEC transaction (#8534) +* Fix missing SLOWLOG records for blocked commands (#8632) +* Allow RESET command during busy scripts (#8629) +* Fix some error replies were not counted on stats (#8659) + +Bug fixes: +* Add a timeout mechanism for replicas stuck in fullsync (#8762) +* Process HELLO command even if the default user has no permissions (#8633) +* Client issuing a long running script and using a pipeline, got disconnected (#8715) +* Fix script kill to work also on scripts that use `pcall` (#8661) +* Fix list-compress-depth may compress more node than required (#8311) +* Fix redis-cli handling of rediss:// URL scheme (#8705) +* Cluster: Skip unnecessary check which may prevent failure detection (#8585) +* Cluster: Fix hang manual failover when replica just started (#8651) +* Sentinel: Fix info-refresh time field before sentinel get first response (#8567) +* Sentinel: Fix possible crash on failed connection attempt (#8627) +* Systemd: Send the readiness notification when a replica is ready to accept connections (#8409) + +Command behavior changes: +* ZADD: fix wrong reply when INCR used with GT/LT which blocked the update (#8717) + It was responding with the incremented value rather than nil +* XAUTOCLAIM: fix response to return the next available id as the cursor (#8725) + Previous behavior was retuning the last one which was already scanned +* XAUTOCLAIM: fix JUSTID to prevent incrementing delivery_count (#8724) + +New config options: +* Add cluster-allow-replica-migration config option (#5285) +* Add replica-announced config option (#8653) +* Add support for plaintext clients in TLS cluster (#8587) +* Add support for reading encrypted keyfiles (#8644) + +Improvements: +* Fix performance regression in BRPOP on Redis 6.0 (#8689) +* Avoid adding slowlog entries for config with sensitive data (#8584) +* Improve redis-cli non-binary safe string handling (#8566) +* Optimize CLUSTER SLOTS reply (#8541) +* Handle remaining fsync errors (#8419) + +Info fields and introspection changes: +* Strip % sign from current_fork_perc info field (#8628) +* Fix RSS memory info on FreeBSD (#8620) +* Fix client_recent_max_input/output_buffer in 'INFO CLIENTS' when all clients drop (#8588) +* Fix invalid master_link_down_since_seconds in info replication (#8785) + +Platform and deployment-related changes: +* Fix FreeBSD <12.x builds (#8603) + +Modules: +* Add macros for RedisModule_log logging levels (#4246) +* Add RedisModule_GetAbsExpire / RedisModule_SetAbsExpire (#8564) +* Add a module type for key space notification (#8759) +* Set module eviction context flag only in masters (#8631) +* Fix unusable RedisModule_IsAOFClient API (#8596) +* Fix missing EXEC on modules propagation after failed EVAL execution (#8654) +* Fix edge-case when a module client is unblocked (#8618) + +================================================================================ +Redis 6.2.1 Released Mon Mar 1 17:51:36 IST 2021 +================================================================================ + +Upgrade urgency: LOW. + +Here is a comprehensive list of changes in this release compared to 6.2.0, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ + +Bug fixes: +* Fix sanitize-dump-payload for stream with deleted records (#8568) +* Prevent client-query-buffer-limit config from being set to lower than 1mb (#8557) + +Improvements: +* Make port, tls-port and bind config options modifiable at runtime (#8510) + +Platform and deployment-related changes: +* Fix compilation error on non-glibc systems if jemalloc is not used (#8533) +* Improved memory consumption and memory usage tracking on FreeBSD (#8545) +* Fix compilation on ARM64 MacOS with jemalloc (#8458) + +Modules: +* New Module API for getting user name of a client (#8508) +* Optimize RM_Call by utilizing a shared reusable client (#8516) +* Fix crash running CLIENT INFO via RM_Call (#8560) + +================================================================================ +Redis 6.2.0 GA Released Tue Feb 22 14:00:00 IST 2021 +================================================================================ + +Upgrade urgency: SECURITY if you use 32bit build of redis (see bellow), MODERATE +if you used earlier versions of Redis 6.2, LOW otherwise. + +Integer overflow on 32-bit systems (CVE-2021-21309): +Redis 4.0 or newer uses a configurable limit for the maximum supported bulk +input size. By default, it is 512MB which is a safe value for all platforms. +If the limit is significantly increased, receiving a large request from a client +may trigger several integer overflow scenarios, which would result with buffer +overflow and heap corruption. + +Here is a comprehensive list of changes in this release compared to 6.2 RC3, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ + +Bug fixes: +* Avoid 32-bit overflows when proto-max-bulk-len is set high (#8522) +* Fix broken protocol in client tracking tracking-redir-broken message (#8456) +* Avoid unsafe field name characters in INFO commandstats, errorstats, modules (#8492) +* XINFO able to access expired keys during CLIENT PAUSE WRITE (#8436) +* Fix allowed length for REPLCONF ip-address, needed due to Sentinel's support for hostnames (#8517) +* Fix broken protocol in redis-benchmark when used with -a or --dbnum (#8486) +* XADD counts deleted records too when considering switching to a new listpack (#8390) + +Bug fixes that are only applicable to previous releases of Redis 6.2: +* Fixes in GEOSEARCH bybox (accuracy and mismatch between width and height) (#8445) +* Fix risk of OOM panic in HRANDFIELD, ZRANDMEMBER commands with huge negative count (#8429) +* Fix duplicate replicas issue in Sentinel, needed due to hostname support (#8481) +* Fix Sentinel configuration rewrite, an improvement of #8271 (#8480) + +Command behavior changes: +* SRANDMEMBER uses RESP3 array type instead of set type (#8504) +* EXPIRE, EXPIREAT, SETEX, GETEX: Return error when provided expire time overflows (#8287) + +Other behavior changes: +* Remove ACL subcommand validation if fully added command exists. (#8483) + +Improvements: +* Optimize sorting in GEORADIUS / GEOSEARCH with COUNT (#8326) +* Optimize HRANDFIELD and ZRANDMEMBER case 4 when ziplist encoded (#8444) +* Optimize in-place replacement of elements in HSET, HINCRBY, LSET (#8493) +* Remove redundant list to store pubsub patterns (#8472) +* Add --insecure option to command line tools (#8416) + +Info fields and introspection changes: +* Add INFO fields to track progress of BGSAVE, AOFRW, replication (#8414) + +Modules: +* RM_ZsetRem: Delete key if empty, the bug could leave empty zset keys (#8453) +* RM_HashSet: Add COUNT_ALL flag and set errno (#8446) + +================================================================================ +Redis 6.2 RC3 Released Tue Feb 1 14:00:00 IST 2021 +================================================================================ + +Upgrade urgency LOW: This is the third Release Candidate of Redis 6.2. + +Here is a comprehensive list of changes in this release compared to 6.2 RC2, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ + +New commands / args: +* Add HRANDFIELD and ZRANDMEMBER commands (#8297) +* Add FAILOVER command (#8315) +* Add GETEX, GETDEL commands (#8327) +* Add PXAT/EXAT arguments to SET command (#8327) +* Add SYNC arg to FLUSHALL and FLUSHDB, and ASYNC/SYNC arg to SCRIPT FLUSH (#8258) + +Sentinel: +* Add hostname support to Sentinel (#8282) +* Prevent file descriptors from leaking into Sentinel scripts (#8242) +* Fix config file line order dependency and config rewrite sequence (#8271) + +New configuration options: +* Add set-proc-title config option to disable changes to the process title (#3623) +* Add proc-title-template option to control what's shown in the process title (#8397) +* Add lazyfree-lazy-user-flush config option to control FLUSHALL, FLUSHDB and SCRIPT FLUSH (#8258) + +Bug fixes: +* AOF: recover from last write error by turning on/off appendonly config (#8030) +* Exit on fsync error when the AOF fsync policy is 'always' (#8347) +* Avoid assertions (on older kernels) when testing arm64 CoW bug (#8405) +* CONFIG REWRITE should honor umask settings (#8371) +* Fix firstkey,lastkey,step in COMMAND command for some commands (#8367) + +Special considerations: +* Fix misleading description of the save configuration directive (#8337) + +Improvements: +* A way to get RDB file via replication without excessive replication buffers (#8303) +* Optimize performance of clusterGenNodesDescription for large clusters (#8182) + +Info fields and introspection changes: +* SLOWLOG and LATENCY monitor include unblocking time of blocked commands (#7491) + +Modules: +* Add modules API for streams (#8288) +* Add event for fork child birth and termination (#8289) +* Add RM_BlockedClientMeasureTime* etc, to track background processing in commandstats (#7491) +* Fix bug in v6.2, wrong value passed to the new unlink callback (#8381) +* Fix bug in v6.2, modules blocked on keys unblock on commands like LPUSH (#8356) + +================================================================================ +Redis 6.2 RC2 Released Tue Jan 12 16:17:20 IST 2021 +================================================================================ + +Upgrade urgency LOW: This is the second Release Candidate of Redis 6.2. + +IMPORTANT: If you're running Redis on ARM64 or a big-endian system, upgrade may +have significant implications. Please be sure to read the notes below. + +Here is a comprehensive list of changes in this release compared to 6.2 RC1, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ + +New commands / args: +* Add the REV, BYLEX and BYSCORE arguments to ZRANGE, and the ZRANGESTORE command (#7844) +* Add the XAUTOCLAIM command (#7973) +* Add the MINID trimming strategy and the LIMIT argument to XADD and XTRIM (#8169) +* Add the ANY argument to GEOSEARCH and GEORADIUS (#8259) +* Add the CH, NX, XX arguments to GEOADD (#8227) +* Add the COUNT argument to LPOP and RPOP (#8179) +* Add the WRITE argument to CLIENT PAUSE for pausing write commands exclusively (#8170) +* Change the proto-ver argument of HELLO to optional (#7377) +* Add the CLIENT TRACKINGINFO subcommand (#7309) + +Command behavior changes: +* CLIENT TRACKING yields an error when given overlapping BCAST prefixes (#8176) +* SWAPDB invalidates WATCHed keys (#8239) +* SORT command behaves differently when used on a writable replica (#8283) + +Other behavior changes: +* Avoid propagating MULTI/EXEC for read-only transactions (#8216) +* Remove the read-only flag from TIME, ECHO, ROLE, LASTSAVE (#8216) +* Fix the command flags of PFDEBUG (#8222) +* Tracking clients will no longer receive unnecessary key invalidation messages after FLUSHDB (#8039) +* Sentinel: Fix missing updates to the config file after SENTINEL SET command (#8229) + +Bug fixes with compatibility implications (bugs introduced in Redis 6.0): +* Fix RDB CRC64 checksum on big-endian systems (#8270) + If you're using big-endian please consider the compatibility implications with + RESTORE, replication and persistence. +* Fix wrong order of key/value in Lua's map response (#8266) + If your scripts use redis.setresp() or return a map (new in Redis 6.0), please + consider the implications. + +Bug fixes that are only applicable to previous releases of Redis 6.2: +* Resolve rare assertions in active defragmentation while loading (#8284, #8281) + +Bug fixes: +* Fix the selection of a random element from large hash tables (#8133) +* Fix an issue where a forked process deletes the parent's pidfile (#8231) +* Fix crashes when enabling io-threads-do-reads (#8230) +* Fix a crash in redis-cli after executing cluster backup (#8267) +* Fix redis-benchmark to use an IP address for the first cluster node (#8154) +* Fix saving of strings larger than 2GB into RDB files (#8306) + +Additional improvements: +* Improve replication handshake time (#8214) +* Release client tracking table memory asynchronously in cases where the DB is also freed asynchronously (#8039) +* Avoid wasteful transient memory allocation in certain cases (#8286, #5954) +* Handle binary string values by the 'requirepass' and 'masterauth' configs (#8200) + +Platform and deployment-related changes: +* Install redis-check-rdb and redis-check-aof as symlinks to redis-server (#5745) +* Add a check for an ARM64 Linux kernel bug (#8224) + Due to the potential severity of this issue, Redis will refuse to run on + affected platforms by default. + +Info fields and introspection changes: +* Add the errorstats section to the INFO command (#8217) +* Add the failed_calls and rejected_calls fields INFO's commandstats section (#8217) +* Report child copy-on-write metrics continuously (#8264) + +Module API changes: +* Add the RedisModule_SendChildCOWInfo API (#8264) +* Add the may-replicate command flag (#8170) + +================================================================================ +Redis 6.2 RC1 Released Mon Dec 14 11:50:00 IST 2020 +================================================================================ + +Upgrade urgency LOW: This is the first Release Candidate of Redis 6.2. + +Introduction to the Redis 6.2 release +===================================== + +This release is the first significant Redis release managed by the core team +under the new project governance model. + +Redis 6.2 includes many new commands and improvements, but no big features. It +mainly makes Redis more complete and addresses issues that have been requested +by many users frequently or for a long time. + +Many of these changes were not eligible for 6.0.x for several reasons: + +1. They are not backward compatible, which is always the case with new or + extended commands (that cannot be replicated to an older replica). +2. They require a longer release-candidate test cycle. + + +Here is a comprehensive list of changes in this release compared to 6.0.9, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ + +New commands / args: +* Add SMISMEMBER command that checks multiple members (#7615) +* Add ZMSCORE command that returns an array of scores (#7593) +* Add LMOVE and BLMOVE commands that pop and push arbitrarily (#6929) +* Add RESET command that resets client connection state (#7982) +* Add COPY command that copies keys (#7953) +* Add ZDIFF and ZDIFFSTORE commands (#7961) +* Add ZINTER and ZUNION commands (#7794) +* Add GEOSEARCH/GEOSEARCHSTORE commands for bounding box spatial queries (#8094) +* Add GET parameter to SET command, for more powerful GETSET (#7852) +* Add exclusive range query to XPENDING (#8130) +* Add exclusive range query to X[REV]RANGE (#8072) +* Add GT and LT options to ZADD for conditional score updates (#7818) +* Add CLIENT INFO and CLIENT LIST for specific ids (#8113) +* Add IDLE argument to XPENDING command (#7972) +* Add local address to CLIENT LIST, and a CLIENT KILL filter. (#7913) +* Add NOMKSTREAM option to XADD command (#7910) +* Add command introspection to Sentinel (#7940) +* Add SENTINEL MYID subcommand (#7858) + +New features: +* Dump payload sanitization: prevent corrupt payload causing crashes (#7807) + Has flags to enable full O(N) validation (disabled by default). +* ACL patterns for Pub/Sub channels (#7993) +* Support ACL for Sentinel mode (#7888) +* Support getting configuration from both stdin and file at the same time (#7893) + Lets you avoid storing secrets on the disk. + +New features in CLI tools: +* redis-cli RESP3 push support (#7609) +* redis-cli cluster import support source and target that require auth (#7994) +* redis-cli URIs able to provide user name in addition to password (#8048) +* redis-cli/redis-benchmark allow specifying the prefered ciphers/ciphersuites (#8005) +* redis-cli add -e option to exit with code when command execution fails (#8136) + +Command behavior changes: +* EXISTS should not alter LRU (#8016) + In Redis 5.0 and 6.0 it would have touched the LRU/LFU of the key. +* OBJECT should not reveal logically expired keys (#8016) + Will now behave the same TYPE or any other non-DEBUG command. +* Improve db id range check for SELECT and MOVE (#8085) + Changes the error message text on a wrong db index. +* Modify AUTH / HELLO error message (#7648) + Changes the error message text when the user isn't found or is disabled. +* BITOPS length limited to proto_max_bulk_len rather than 512MB (#8096) + The limit is now configurable like in SETRANGE, and APPEND. +* GEORADIUS[BYMEMBER] can fail with -OOM if Redis is over the memory limit (#8107) + +Other behavior changes: +* Optionally (default) fail to start if requested bind address is not available (#7936) + If you rely on Redis starting successfully even if one of the bind addresses + is not available, you'll need to tune the new config. +* Limit the main db dictionaries expansion to prevent key eviction (#7954) + In the past big dictionary rehashing could result in massive data eviction. + Now this rehashing is delayed (up to a limit), which can result in performance + loss due to hash collisions. +* CONFIG REWRITE is atomic and safer, but requires write access to the config file's folder (#7824, #8051) + This change was already present in 6.0.9, but was missing from the release + notes. +* A new incremental eviction mechanism that reduces latency on eviction spikes (#7653) + In pathological cases this can cause memory to grow uncontrolled and may require + specific tuning. +* Not resetting "save" config when Redis is started with command line arguments. (#7092) + In case you provide command line arguments without "save" and count on it + being disabled, Now the defaults "save" config will kick in. +* Update memory metrics for INFO during loading (#7690) +* When "supervised" config is enabled, it takes precedence over "daemonize". (#8036) +* Assertion and panic, print crash log without generating SIGSEGV (#7585) +* Added crash log report on SIGABRT, instead of silently exiting (#8004) +* Disable THP (Transparent Huge Pages) if enabled (#7381) + If you deliberately enabled it, you'll need to config Redis to keep it. + +Bug fixes: +* Handle output buffer limits for module blocked clients (#8141) + Could result in a module sending reply to a blocked client to go beyond the + limit. +* Fix setproctitle related crashes. (#8150, #8088) + Caused various crashes on startup, mainly on Apple M1 chips or under + instrumentation. +* A module doing RM_Call could cause replicas to get nested MULTI (#8097). +* Backup/restore cluster mode keys to slots map for repl-diskless-load=swapdb (#8108) + In cluster mode with repl-diskless-load, when loading failed, slot map + wouldn't have been restored. +* Fix oom-score-adj-values range, and bug when used in config file (#8046) + Enabling setting this in the config file in a line after enabling it, would + have been buggy. +* Reset average ttl when empty databases (#8106) + Just causing misleading metric in INFO +* Disable rehash when Redis has child process (#8007) + This could have caused excessive CoW during BGSAVE, replication or AOFRW. +* Further improved ACL algorithm for picking categories (#7966) + Output of ACL GETUSER is now more similar to the one provided by ACL SETUSER. +* Fix bug with module GIL being released prematurely (#8061) + Could in theory (and rarely) cause multi-threaded modules to corrupt memory. +* Fix cluster redirect for module command with no firstkey. (#7539) +* Reduce effect of client tracking causing feedback loop in key eviction (#8100) +* Kill disk-based fork child when all replicas drop and 'save' is not enabled (#7819) +* Rewritten commands (modified for propagation) are logged as their original command (#8006) +* Fix cluster access to unaligned memory (SIGBUS on old ARM) #7958 +* If diskless repl child is killed, make sure to reap the child pid (#7742) +* Broadcast a PONG message when slot's migration is over, may reduce MOVED responses (#7571) + +Other improvements: +* TLS Support in redis-benchmark (#7959) +* Accelerate diskless master connections, and general re-connections (#6271) +* Run active defrag while blocked / loading (#7726) +* Performance and memory reporting improvement - sds take control of its internal fragmentation (#7875) +* Speedup cluster failover. (#7948) + +Platform / toolchain support related improvements: +* Optionally (not by default) use H/W Monotonic clock for faster time sampling (#7644) +* Remove the requirements for C11 and _Atomic supporting compiler (#7707) + This would allow to more easily build and use Redis on older systems and + compilers again. +* Fix crash log registers output on ARM. (#8020) +* Raspberry build fix. (#8095) +* Setting process title support for Haiku. (#8060) +* DragonFlyBSD RSS memory sampling support. (#8023) + +New configuration options: +* Enable configuring OpenSSL using the standard openssl.cnf (#8143) +* oom-score-adj-values config can now take absolute values (besides relative ones) (#8046) +* TLS: Add different client cert support. (#8076) +* Note that a few other changes listed above added their config options. + +Info fields and introspection changes: +* Add INFO fields to track diskless and disk-based replication progress (#7981) +* Add INFO field for main thread cpu time, and scrape system time. (#8132) +* Add total_forks to INFO STATS (#8155) +* Add maxclients and cluster_connections to INFO CLIENTS (#7979) +* Add tracking bcast flag and client redirection in client list (#7995) +* Fixed INFO client_recent_max_input_buffer includes argv array (#8065, see #7874) +* Note that a few other changes listed above added their info fields. + +Module API changes: +* Add CTX_FLAGS_DENY_BLOCKING as a unified the way to know if blocking is allowed (#8025) +* Add data type callbacks for lazy free effort, and unlink (#7912) +* Add data type callback for COPY command (#8112) +* Add callbacks for defrag support. (#8149) +* Add module event for repl-diskless-load swapdb (#8153) + +Module related fixes: +* Moved RMAPI_FUNC_SUPPORTED so that it's usable (#8037) +* Improve timer accuracy (#7987) +* Allow '\0' inside of result of RM_CreateStringPrintf (#6260) + + +Thanks to all the users and developers who made this release possible. +We'll follow up with more RC releases, until the code looks production ready +and we don't get reports of serious issues for a while. + +A special thank you for the amount of work put into this release by: +- Oran Agra +- Yossi Gottlieb +- Viktor Söderqvist +- Yang Bodong +- Filipe Oliveira +- Guy Benoish +- Itamar Haber +- Madelyn Olson +- Wang Yuan +- Felipe Machado +- Wen Hui +- Tatsuya Arisawa +- Jonah H. Harris +- Raghav Muddur +- Jim Brunner +- Yaacov Hazan +- Allen Farris +- Chen Yang +- Nitai Caro +- sundb +- Meir Shpilraien +- maohuazhu +- Valentino Geron +- Zhao Zhao +- Qu Chen +- George Prekas +- Tyson Andre +- Uri Yagelnik +- Michael Grunder +- Huang Zw +- alexronke-channeladvisor +- Andy Pan +- Wu Yunlong +- Wei Kukey +- Yoav Steinberg +- Greg Femec +- Uri Shachar +- Nykolas Laurentino de Lima +- xhe +- zhenwei pi +- David CARLIER + +Migrating from 6.0 to 6.2 +========================= + +Redis 6.2 is mostly a strict superset of 6.0, you should not have any problem +upgrading your application from 6.0 to 6.2. However there are some small changes +of behavior listed above, please make sure you are not badly affected by any of +them. + +Specifically these sections: +* Command behavior changes +* Other behavior changes + +-------------------------------------------------------------------------------- + +Cheers, +The Redis team diff --git a/pkg/deb/conf/keydb.conf b/pkg/deb/conf/keydb.conf index 22844de5a..8e1552b9a 100644 --- a/pkg/deb/conf/keydb.conf +++ b/pkg/deb/conf/keydb.conf @@ -150,6 +150,11 @@ tcp-keepalive 300 # # tls-cert-file keydb.crt # tls-key-file keydb.key +# +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-key-file-pass secret # Normally Redis uses the same certificate for both server functions (accepting # connections) and client functions (replicating from a master, establishing @@ -162,6 +167,11 @@ tcp-keepalive 300 # # tls-client-cert-file client.crt # tls-client-key-file client.key +# +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-client-key-file-pass secret # Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: # @@ -659,6 +669,18 @@ repl-disable-tcp-nodelay no # By default the priority is 100. replica-priority 100 +# ----------------------------------------------------------------------------- +# By default, Redis Sentinel includes all replicas in its reports. A replica +# can be excluded from Redis Sentinel's announcements. An unannounced replica +# will be ignored by the 'sentinel replicas ' command and won't be +# exposed to Redis Sentinel's clients. +# +# This option does not change the behavior of replica-priority. Even with +# replica-announced set to 'no', the replica can be promoted to master. To +# prevent this behavior, set replica-priority to 0. +# +# replica-announced yes + # It is possible for a master to stop accepting writes if there are less than # N replicas connected, having a lag less or equal than M seconds. # @@ -897,7 +919,7 @@ acllog-max-len 128 # order to provide better out-of-the-box Pub/Sub security. Therefore, it is # recommended that you explicitly define Pub/Sub permissions for all users # rather then rely on implicit default values. Once you've set explicit -# Pub/Sub for all exisitn users, you should uncomment the following line. +# Pub/Sub for all existing users, you should uncomment the following line. # # acl-pubsub-default resetchannels @@ -1227,7 +1249,7 @@ disable-thp yes # If the AOF is enabled on startup Redis will load the AOF, that is the file # with the better durability guarantees. # -# Please check http://redis.io/topics/persistence for more information. +# Please check https://redis.io/topics/persistence for more information. appendonly no @@ -1442,12 +1464,21 @@ lua-time-limit 5000 # master in your cluster. # # Default is 1 (replicas migrate only if their masters remain with at least -# one replica). To disable migration just set it to a very large value. +# one replica). To disable migration just set it to a very large value or +# set cluster-allow-replica-migration to 'no'. # A value of 0 can be set but is useful only for debugging and dangerous # in production. # # cluster-migration-barrier 1 +# Turning off this option allows to use less automatic cluster configuration. +# It both disables migration to orphaned masters and migration from masters +# that became empty. +# +# Default is 'yes' (allow automatic migrations). +# +# cluster-allow-replica-migration yes + # By default Redis Cluster nodes stop accepting queries if they detect there # is at least a hash slot uncovered (no available node is serving it). # This way if the cluster is partially down (for example a range of hash slots @@ -1472,7 +1503,7 @@ lua-time-limit 5000 # cluster-replica-no-failover no # In order to setup your cluster make sure to read the documentation -# available at http://redis.io web site. +# available at https://redis.io web site. ########################## CLUSTER DOCKER/NAT support ######################## @@ -1482,16 +1513,21 @@ lua-time-limit 5000 # # In order to make Redis Cluster working in such environments, a static # configuration where each node knows its public address is needed. The -# following two options are used for this scope, and are: +# following four options are used for this scope, and are: # # * cluster-announce-ip # * cluster-announce-port +# * cluster-announce-tls-port # * cluster-announce-bus-port # -# Each instructs the node about its address, client port, and cluster message -# bus port. The information is then published in the header of the bus packets -# so that other nodes will be able to correctly map the address of the node -# publishing the information. +# Each instructs the node about its address, client ports (for connections +# without and with TLS) and cluster message bus port. The information is then +# published in the header of the bus packets so that other nodes will be able to +# correctly map the address of the node publishing the information. +# +# If cluster-tls is set to yes and cluster-announce-tls-port is omitted or set +# to zero, then cluster-announce-port refers to the TLS port. Note also that +# cluster-announce-tls-port has no effect if cluster-tls is set to no. # # If the above options are not used, the normal Redis Cluster auto-detection # will be used instead. @@ -1504,7 +1540,8 @@ lua-time-limit 5000 # Example: # # cluster-announce-ip 10.1.1.5 -# cluster-announce-port 6379 +# cluster-announce-tls-port 6379 +# cluster-announce-port 0 # cluster-announce-bus-port 6380 ################################## SLOW LOG ################################### @@ -1555,7 +1592,7 @@ latency-monitor-threshold 0 ############################# EVENT NOTIFICATION ############################## # Redis can notify Pub/Sub clients about events happening in the key space. -# This feature is documented at http://redis.io/topics/notifications +# This feature is documented at https://redis.io/topics/notifications # # For instance if keyspace events notification is enabled, and a client # performs a DEL operation on key "foo" stored in the Database 0, two @@ -1578,8 +1615,9 @@ latency-monitor-threshold 0 # x Expired events (events generated every time a key expires) # e Evicted events (events generated when a key is evicted for maxmemory) # t Stream commands +# d Module key type events # m Key-miss events (Note: It is not included in the 'A' class) -# A Alias for g$lshzxet, so that the "AKE" string means all the events +# A Alias for g$lshzxetd, so that the "AKE" string means all the events # (Except key-miss events which are excluded from 'A' due to their # unique nature). # diff --git a/src/Makefile b/src/Makefile index 43c861acd..f0dea3dac 100644 --- a/src/Makefile +++ b/src/Makefile @@ -301,6 +301,17 @@ endif FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a $(LIBSSL_LIBS) $(LIBCRYPTO_LIBS) endif +ifndef V + define MAKE_INSTALL + @printf ' %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$(1)$(ENDCOLOR) 1>&2 + @$(INSTALL) $(1) $(2) + endef +else + define MAKE_INSTALL + $(INSTALL) $(1) $(2) + endef +endif + REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) REDIS_CXX=$(QUIET_CC)$(CXX) $(FINAL_CXXFLAGS) KEYDB_AS=$(QUIET_CC) as --64 -g @@ -402,9 +413,6 @@ $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ) $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/hdr_histogram/hdr_histogram.o $(FINAL_LIBS) -dict-benchmark: dict.cpp zmalloc.cpp sds.c siphash.c mt19937-64.c - $(REDIS_CC) $(FINAL_CFLAGS) $^ -D DICT_BENCHMARK_MAIN -o $@ $(FINAL_LIBS) - DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d) -include $(DEP) @@ -421,7 +429,7 @@ DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ $(KEYDB_AS) $< -o $@ clean: - rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov KeyDB.info lcov-html Makefile.dep dict-benchmark + rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov KeyDB.info lcov-html Makefile.dep rm -f $(DEP) .PHONY: clean @@ -476,9 +484,9 @@ src/help.h: install: all @mkdir -p $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_BENCHMARK_NAME) $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_CLI_NAME) $(INSTALL_BIN) + $(call MAKE_INSTALL,$(REDIS_SERVER_NAME),$(INSTALL_BIN)) + $(call MAKE_INSTALL,$(REDIS_BENCHMARK_NAME),$(INSTALL_BIN)) + $(call MAKE_INSTALL,$(REDIS_CLI_NAME),$(INSTALL_BIN)) @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_CHECK_RDB_NAME) @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_CHECK_AOF_NAME) @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_SENTINEL_NAME) diff --git a/src/acl.cpp b/src/acl.cpp index 1a35b5bd9..654a9d6a2 100644 --- a/src/acl.cpp +++ b/src/acl.cpp @@ -248,7 +248,7 @@ user *ACLCreateUser(const char *name, size_t namelen) { if (raxFind(Users,(unsigned char*)name,namelen) != raxNotFound) return NULL; user *u = (user*)zmalloc(sizeof(*u), MALLOC_LOCAL); u->name = sdsnewlen(name,namelen); - u->flags = USER_FLAG_DISABLED | g_pserver->acl_pubusub_default; + u->flags = USER_FLAG_DISABLED | g_pserver->acl_pubsub_default; u->allowed_subcommands = NULL; u->passwords = listCreate(); u->patterns = listCreate(); @@ -655,6 +655,7 @@ sds ACLDescribeUser(user *u) { if (u->flags & USER_FLAG_ALLCHANNELS) { res = sdscatlen(res,"&* ",3); } else { + res = sdscatlen(res,"resetchannels ",14); listRewind(u->channels,&li); while((ln = listNext(&li))) { sds thispat = (sds)listNodeValue(ln); @@ -1003,6 +1004,8 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { serverAssert(ACLSetUser(u,"resetpass",-1) == C_OK); serverAssert(ACLSetUser(u,"resetkeys",-1) == C_OK); serverAssert(ACLSetUser(u,"resetchannels",-1) == C_OK); + if (g_pserver->acl_pubsub_default & USER_FLAG_ALLCHANNELS) + serverAssert(ACLSetUser(u,"allchannels",-1) == C_OK); serverAssert(ACLSetUser(u,"off",-1) == C_OK); serverAssert(ACLSetUser(u,"sanitize-payload",-1) == C_OK); serverAssert(ACLSetUser(u,"-@all",-1) == C_OK); @@ -1183,9 +1186,9 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) { /* If there is no associated user, the connection can run anything. */ if (u == NULL) return ACL_OK; - /* Check if the user can execute this command. */ - if (!(u->flags & USER_FLAG_ALLCOMMANDS) && - c->cmd->proc != authCommand) + /* Check if the user can execute this command or if the command + * doesn't need to be authenticated (hello, auth). */ + if (!(u->flags & USER_FLAG_ALLCOMMANDS) && !(c->cmd->flags & CMD_NO_AUTH)) { /* If the bit is not set we have to check further, in case the * command is allowed just with that specific subcommand. */ @@ -1363,6 +1366,22 @@ int ACLCheckPubsubPerm(client *c, int idx, int count, int literal, int *idxptr) } +/* Check whether the command is ready to be exceuted by ACLCheckCommandPerm. + * If check passes, then check whether pub/sub channels of the command is + * ready to be executed by ACLCheckPubsubPerm */ +int ACLCheckAllPerm(client *c, int *idxptr) { + int acl_retval = ACLCheckCommandPerm(c,idxptr); + if (acl_retval != ACL_OK) + return acl_retval; + if (c->cmd->proc == publishCommand) + acl_retval = ACLCheckPubsubPerm(c,1,1,0,idxptr); + else if (c->cmd->proc == subscribeCommand) + acl_retval = ACLCheckPubsubPerm(c,1,c->argc-1,0,idxptr); + else if (c->cmd->proc == psubscribeCommand) + acl_retval = ACLCheckPubsubPerm(c,1,c->argc-1,1,idxptr); + return acl_retval; +} + /* ============================================================================= * ACL loading / saving functions * ==========================================================================*/ @@ -1876,6 +1895,10 @@ void addACLLogEntry(client *c, int reason, int argpos, sds username) { void aclCommand(client *c) { char *sub = szFromObj(c->argv[1]); if (!strcasecmp(sub,"setuser") && c->argc >= 3) { + /* Consider information about passwords or permissions + * to be sensitive, which will be the arguments for this + * subcommand. */ + preventCommandLogging(c); sds username = szFromObj(c->argv[2]); /* Check username validity. */ if (ACLStringHasSpaces(username,sdslen(username))) { diff --git a/src/ae.cpp b/src/ae.cpp index abe6b5bb7..efd9b0cb1 100644 --- a/src/ae.cpp +++ b/src/ae.cpp @@ -491,7 +491,7 @@ extern "C" int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id) return AE_ERR; /* NO event with the specified ID found */ } -/* How many milliseconds until the first timer should fire. +/* How many microseconds until the first timer should fire. * If there are no timers, -1 is returned. * * Note that's O(N) since time events are unsorted. @@ -500,7 +500,7 @@ extern "C" int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id) * Much better but still insertion or deletion of timers is O(N). * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)). */ -static long msUntilEarliestTimer(aeEventLoop *eventLoop) { +static int64_t usUntilEarliestTimer(aeEventLoop *eventLoop) { aeTimeEvent *te = eventLoop->timeEventHead; if (te == NULL) return -1; @@ -512,8 +512,7 @@ static long msUntilEarliestTimer(aeEventLoop *eventLoop) { } monotime now = getMonotonicUs(); - return (now >= earliest->when) - ? 0 : (long)((earliest->when - now) / 1000); + return (now >= earliest->when) ? 0 : earliest->when - now; } /* Process time events */ @@ -674,14 +673,14 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) { int j; struct timeval tv, *tvp; - long msUntilTimer = -1; + int64_t usUntilTimer = -1; if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT)) - msUntilTimer = msUntilEarliestTimer(eventLoop); + usUntilTimer = usUntilEarliestTimer(eventLoop); - if (msUntilTimer >= 0) { - tv.tv_sec = msUntilTimer / 1000; - tv.tv_usec = (msUntilTimer % 1000) * 1000; + if (usUntilTimer >= 0) { + tv.tv_sec = usUntilTimer / 1000000; + tv.tv_usec = usUntilTimer % 1000000; tvp = &tv; } else { /* If we have to check for events but need to return diff --git a/src/ae_epoll.cpp b/src/ae_epoll.cpp index 67c512063..76cfa680a 100644 --- a/src/ae_epoll.cpp +++ b/src/ae_epoll.cpp @@ -115,7 +115,7 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { int retval, numevents = 0; retval = epoll_wait(state->epfd,state->events,eventLoop->setsize, - tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1); + tvp ? (tvp->tv_sec*1000 + (tvp->tv_usec + 999)/1000) : -1); if (retval > 0) { int j; diff --git a/src/anet.c b/src/anet.c index dcd0a1347..0b3e462c3 100644 --- a/src/anet.c +++ b/src/anet.c @@ -186,27 +186,6 @@ int anetDisableTcpNoDelay(char *err, int fd) return anetSetTcpNoDelay(err, fd, 0); } - -int anetSetSendBuffer(char *err, int fd, int buffsize) -{ - if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &buffsize, sizeof(buffsize)) == -1) - { - anetSetError(err, "setsockopt SO_SNDBUF: %s", strerror(errno)); - return ANET_ERR; - } - return ANET_OK; -} - -int anetTcpKeepAlive(char *err, int fd) -{ - int yes = 1; - if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes)) == -1) { - anetSetError(err, "setsockopt SO_KEEPALIVE: %s", strerror(errno)); - return ANET_ERR; - } - return ANET_OK; -} - /* Set the socket send timeout (SO_SNDTIMEO socket option) to the specified * number of milliseconds, or disable it if the 'ms' argument is zero. */ int anetSendTimeout(char *err, int fd, long long ms) { @@ -392,23 +371,11 @@ end: } } -int anetTcpConnect(char *err, const char *addr, int port) -{ - return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONE); -} - int anetTcpNonBlockConnect(char *err, const char *addr, int port) { return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONBLOCK); } -int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, - const char *source_addr) -{ - return anetTcpGenericConnect(err,addr,port,source_addr, - ANET_CONNECT_NONBLOCK); -} - int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, const char *source_addr) { @@ -444,46 +411,6 @@ int anetUnixGenericConnect(char *err, const char *path, int flags) return s; } -int anetUnixConnect(char *err, const char *path) -{ - return anetUnixGenericConnect(err,path,ANET_CONNECT_NONE); -} - -int anetUnixNonBlockConnect(char *err, const char *path) -{ - return anetUnixGenericConnect(err,path,ANET_CONNECT_NONBLOCK); -} - -/* Like read(2) but make sure 'count' is read before to return - * (unless error or EOF condition is encountered) */ -int anetRead(int fd, char *buf, int count) -{ - ssize_t nread, totlen = 0; - while(totlen != count) { - nread = read(fd,buf,count-totlen); - if (nread == 0) return totlen; - if (nread == -1) return -1; - totlen += nread; - buf += nread; - } - return totlen; -} - -/* Like write(2) but make sure 'count' is written before to return - * (unless error is encountered) */ -int anetWrite(int fd, char *buf, int count) -{ - ssize_t nwritten, totlen = 0; - while(totlen != count) { - nwritten = write(fd,buf,count-totlen); - if (nwritten == 0) return totlen; - if (nwritten == -1) return -1; - totlen += nwritten; - buf += nwritten; - } - return totlen; -} - static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len, int backlog) { if (bind(s,sa,len) == -1) { anetSetError(err, "bind: %s", strerror(errno)); diff --git a/src/anet.h b/src/anet.h index aaabc9105..0f6cf31ba 100644 --- a/src/anet.h +++ b/src/anet.h @@ -57,26 +57,19 @@ extern "C" { #define FD_TO_PEER_NAME 0 #define FD_TO_SOCK_NAME 1 -int anetTcpConnect(char *err, const char *addr, int port); int anetTcpNonBlockConnect(char *err, const char *addr, int port); -int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, const char *source_addr); int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, const char *source_addr); -int anetUnixConnect(char *err, const char *path); -int anetUnixNonBlockConnect(char *err, const char *path); -int anetRead(int fd, char *buf, int count); int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, int flags); int anetTcpServer(char *err, int port, const char *bindaddr, int backlog, int fReusePort, int fFirstListen); int anetTcp6Server(char *err, int port, const char *bindaddr, int backlog, int fReusePort, int fFirstListen); int anetUnixServer(char *err, char *path, mode_t perm, int backlog); int anetTcpAccept(char *err, int serversock, char *ip, size_t ip_len, int *port); int anetUnixAccept(char *err, int serversock); -int anetWrite(int fd, char *buf, int count); int anetNonBlock(char *err, int fd); int anetBlock(char *err, int fd); int anetCloexec(int fd); int anetEnableTcpNoDelay(char *err, int fd); int anetDisableTcpNoDelay(char *err, int fd); -int anetTcpKeepAlive(char *err, int fd); int anetSendTimeout(char *err, int fd, long long ms); int anetRecvTimeout(char *err, int fd, long long ms); int anetFdToString(int fd, char *ip, size_t ip_len, int *port, int fd_to_str_type); diff --git a/src/aof.cpp b/src/aof.cpp index ec1046947..9ddc52d0a 100644 --- a/src/aof.cpp +++ b/src/aof.cpp @@ -233,7 +233,7 @@ void killAppendOnlyChild(void) { serverLog(LL_NOTICE,"Killing running AOF rewrite child: %ld", (long) g_pserver->child_pid); if (kill(g_pserver->child_pid,SIGUSR1) != -1) { - while(wait3(&statloc,0,NULL) != g_pserver->child_pid); + while(waitpid(-1, &statloc, 0) != g_pserver->child_pid); } /* Reset the buffer accumulating changes while the child saves. */ aofRewriteBufferReset(); @@ -249,9 +249,12 @@ void killAppendOnlyChild(void) { void stopAppendOnly(void) { serverAssert(g_pserver->aof_state != AOF_OFF); flushAppendOnlyFile(1); - redis_fsync(g_pserver->aof_fd); - g_pserver->aof_fsync_offset = g_pserver->aof_current_size; - g_pserver->aof_last_fsync = g_pserver->unixtime; + if (redis_fsync(g_pserver->aof_fd) == -1) { + serverLog(LL_WARNING,"Fail to fsync the AOF file: %s",strerror(errno)); + } else { + g_pserver->aof_fsync_offset = g_pserver->aof_current_size; + g_pserver->aof_last_fsync = g_pserver->unixtime; + } close(g_pserver->aof_fd); g_pserver->aof_fd = -1; @@ -305,6 +308,15 @@ int startAppendOnly(void) { g_pserver->aof_last_fsync = g_pserver->unixtime; g_pserver->aof_fd = newfd; + /* If AOF fsync error in bio job, we just ignore it and log the event. */ + int aof_bio_fsync_status; + atomicGet(g_pserver->aof_bio_fsync_status, aof_bio_fsync_status); + if (aof_bio_fsync_status == C_ERR) { + serverLog(LL_WARNING, + "AOF reopen, just ignore the AOF fsync error in bio job"); + atomicSet(g_pserver->aof_bio_fsync_status,C_OK); + } + /* If AOF was in error state, we just ignore it and log the event. */ if (g_pserver->aof_last_write_status == C_ERR) { serverLog(LL_WARNING,"AOF reopen, just ignore the last error."); @@ -1695,7 +1707,7 @@ int rewriteAppendOnlyFile(char *filename) { if (write(g_pserver->aof_pipe_write_ack_to_parent,"!",1) != 1) goto werr; if (anetNonBlock(NULL,g_pserver->aof_pipe_read_ack_from_parent) != ANET_OK) goto werr; - /* We read the ACK from the server using a 10 seconds timeout. Normally + /* We read the ACK from the server using a 5 seconds timeout. Normally * it should reply ASAP, but just in case we lose its reply, we are sure * the child will eventually get terminated. */ if (syncRead(g_pserver->aof_pipe_read_ack_from_parent,&byte,1,5000) != 1 || diff --git a/src/bio.cpp b/src/bio.cpp index 17ca56d34..b9939c01c 100644 --- a/src/bio.cpp +++ b/src/bio.cpp @@ -220,7 +220,23 @@ void *bioProcessBackgroundJobs(void *arg) { if (type == BIO_CLOSE_FILE) { close(job->fd); } else if (type == BIO_AOF_FSYNC) { - redis_fsync(job->fd); + /* The fd may be closed by main thread and reused for another + * socket, pipe, or file. We just ignore these errno because + * aof fsync did not really fail. */ + if (redis_fsync(job->fd) == -1 && + errno != EBADF && errno != EINVAL) + { + int last_status; + atomicGet(g_pserver->aof_bio_fsync_status,last_status); + atomicSet(g_pserver->aof_bio_fsync_status,C_ERR); + atomicSet(g_pserver->aof_bio_fsync_errno,errno); + if (last_status == C_OK) { + serverLog(LL_WARNING, + "Fail to fsync the AOF file: %s",strerror(errno)); + } + } else { + atomicSet(g_pserver->aof_bio_fsync_status,C_OK); + } } else if (type == BIO_LAZY_FREE) { job->free_fn(job->free_args); } else { diff --git a/src/blocked.cpp b/src/blocked.cpp index c4ca8492c..c6d157a9c 100644 --- a/src/blocked.cpp +++ b/src/blocked.cpp @@ -108,12 +108,11 @@ void blockClient(client *c, int btype) { void updateStatsOnUnblock(client *c, long blocked_us, long reply_us){ const ustime_t total_cmd_duration = c->duration + blocked_us + reply_us; c->lastcmd->microseconds += total_cmd_duration; + /* Log the command into the Slow log if needed. */ - if (!(c->lastcmd->flags & CMD_SKIP_SLOWLOG)) { - slowlogPushEntryIfNeeded(c,c->argv,c->argc,total_cmd_duration); - /* Log the reply duration event. */ - latencyAddSampleIfNeeded("command-unblocking",reply_us/1000); - } + slowlogPushCurrentCommand(c, c->lastcmd, total_cmd_duration); + /* Log the reply duration event. */ + latencyAddSampleIfNeeded("command-unblocking",reply_us/1000); } /* This function is called in the beforeSleep() function of the event loop @@ -134,7 +133,7 @@ void processUnblockedClients(int iel) { listDelNode(unblocked_clients,ln); AssertCorrectThread(c); - fastlock_lock(&c->lock); + std::unique_lock ul(c->lock); c->flags &= ~CLIENT_UNBLOCKED; /* Process remaining data in the input buffer, unless the client @@ -151,7 +150,6 @@ void processUnblockedClients(int iel) { processInputBuffer(c, CMD_CALL_FULL); } } - fastlock_unlock(&c->lock); } } @@ -203,6 +201,16 @@ void unblockClient(client *c) { } else { serverPanic("Unknown btype in unblockClient()."); } + + /* Reset the client for a new query since, for blocking commands + * we do not do it immediately after the command returns (when the + * client got blocked) in order to be still able to access the argument + * vector from module callbacks and updateStatsOnUnblock. */ + if (c->btype != BLOCKED_PAUSE) { + freeClientOriginalArgv(c); + resetClient(c); + } + /* Clear the flags, and put the client in the unblocked list so that * we'll process new commands in its query buffer ASAP. */ g_pserver->blocked_clients--; @@ -246,7 +254,7 @@ void disconnectAllBlockedClients(void) { while((ln = listNext(&li))) { client *c = (client*)listNodeValue(ln); - fastlock_lock(&c->lock); + std::unique_lock ul(c->lock); if (c->flags & CLIENT_BLOCKED) { /* PAUSED clients are an exception, when they'll be unblocked, the * command processing will start from scratch, and the command will @@ -261,7 +269,6 @@ void disconnectAllBlockedClients(void) { unblockClient(c); c->flags |= CLIENT_CLOSE_AFTER_REPLY; } - fastlock_unlock(&c->lock); } } @@ -298,7 +305,6 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) { * freed by the next unblockClient() * call. */ if (dstkey) incrRefCount(dstkey); - unblockClient(receiver); monotime replyTimer; elapsedStart(&replyTimer); @@ -311,6 +317,7 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) { listTypePush(o,value,wherefrom); } updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + unblockClient(receiver); if (dstkey) decrRefCount(dstkey); decrRefCount(value); @@ -355,11 +362,11 @@ void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { int where = (receiver->lastcmd && receiver->lastcmd->proc == bzpopminCommand) ? ZSET_MIN : ZSET_MAX; - unblockClient(receiver); monotime replyTimer; elapsedStart(&replyTimer); genericZpopCommand(receiver,&rl->key,1,where,1,NULL); updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + unblockClient(receiver); zcard--; /* Replicate the command. */ @@ -492,6 +499,10 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { void serveClientsBlockedOnKeyByModule(readyList *rl) { dictEntry *de; + /* Optimization: If no clients are in type BLOCKED_MODULE, + * we can skip this loop. */ + if (!g_pserver->blocked_clients_by_type[BLOCKED_MODULE]) return; + /* We serve clients in the same order they blocked for * this key, from the first blocked to the last. */ de = dictFind(rl->db->blocking_keys,rl->key); @@ -575,7 +586,7 @@ void handleClientsBlockedOnKeys(void) { * way we can lookup an object multiple times (BLMOVE does * that) without the risk of it being freed in the second * lookup, invalidating the first one. - * See https://github.com/antirez/redis/pull/6554. */ + * See https://github.com/redis/redis/pull/6554. */ serverTL->fixed_time_expire++; updateCachedTime(0); diff --git a/src/childinfo.cpp b/src/childinfo.cpp index 9e81d395a..1d763729e 100644 --- a/src/childinfo.cpp +++ b/src/childinfo.cpp @@ -33,6 +33,7 @@ typedef struct { size_t keys; size_t cow; + monotime cow_updated; double progress; childInfoType information_type; /* Type of information */ } child_info_data; @@ -69,18 +70,39 @@ void closeChildInfoPipe(void) { void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, const char *pname) { if (g_pserver->child_info_pipe[1] == -1) return; - child_info_data data = {0}; /* zero everything, including padding to sattisfy valgrind */ + static monotime cow_updated = 0; + static uint64_t cow_update_cost = 0; + static size_t cow = 0; + + child_info_data data = {0}; /* zero everything, including padding to satisfy valgrind */ + + /* When called to report current info, we need to throttle down CoW updates as they + * can be very expensive. To do that, we measure the time it takes to get a reading + * and schedule the next reading to happen not before time*CHILD_COW_COST_FACTOR + * passes. */ + + monotime now = getMonotonicUs(); + if (info_type != CHILD_INFO_TYPE_CURRENT_INFO || + !cow_updated || + now - cow_updated > cow_update_cost * CHILD_COW_DUTY_CYCLE) + { + cow = zmalloc_get_private_dirty(-1); + cow_updated = getMonotonicUs(); + cow_update_cost = cow_updated - now; + + if (cow) { + serverLog((info_type == CHILD_INFO_TYPE_CURRENT_INFO) ? LL_VERBOSE : LL_NOTICE, + "%s: %zu MB of memory used by copy-on-write", + pname, data.cow / (1024 * 1024)); + } + } + data.information_type = info_type; data.keys = keys; - data.cow = zmalloc_get_private_dirty(-1); + data.cow = cow; + data.cow_updated = cow_updated; data.progress = progress; - if (data.cow) { - serverLog((info_type == CHILD_INFO_TYPE_CURRENT_INFO) ? LL_VERBOSE : LL_NOTICE, - "%s: %zu MB of memory used by copy-on-write", - pname, data.cow/(1024*1024)); - } - ssize_t wlen = sizeof(data); if (write(g_pserver->child_info_pipe[1], &data, wlen) != wlen) { @@ -89,9 +111,10 @@ void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, } /* Update Child info. */ -void updateChildInfo(childInfoType information_type, size_t cow, size_t keys, double progress) { +void updateChildInfo(childInfoType information_type, size_t cow, monotime cow_updated, size_t keys, double progress) { if (information_type == CHILD_INFO_TYPE_CURRENT_INFO) { g_pserver->stat_current_cow_bytes = cow; + g_pserver->stat_current_cow_updated = cow_updated; g_pserver->stat_current_save_keys_processed = keys; if (progress != -1) g_pserver->stat_module_progress = progress; } else if (information_type == CHILD_INFO_TYPE_AOF_COW_SIZE) { @@ -107,7 +130,7 @@ void updateChildInfo(childInfoType information_type, size_t cow, size_t keys, do * if complete data read into the buffer, * data is stored into *buffer, and returns 1. * otherwise, the partial data is left in the buffer, waiting for the next read, and returns 0. */ -int readChildInfo(childInfoType *information_type, size_t *cow, size_t *keys, double* progress) { +int readChildInfo(childInfoType *information_type, size_t *cow, monotime *cow_updated, size_t *keys, double* progress) { /* We are using here a static buffer in combination with the server.child_info_nread to handle short reads */ static child_info_data buffer; ssize_t wlen = sizeof(buffer); @@ -124,6 +147,7 @@ int readChildInfo(childInfoType *information_type, size_t *cow, size_t *keys, do if (g_pserver->child_info_nread == wlen) { *information_type = buffer.information_type; *cow = buffer.cow; + *cow_updated = buffer.cow_updated; *keys = buffer.keys; *progress = buffer.progress; return 1; @@ -137,12 +161,13 @@ void receiveChildInfo(void) { if (g_pserver->child_info_pipe[0] == -1) return; size_t cow; + monotime cow_updated; size_t keys; double progress; childInfoType information_type; /* Drain the pipe and update child info so that we get the final message. */ - while (readChildInfo(&information_type, &cow, &keys, &progress)) { - updateChildInfo(information_type, cow, keys, progress); + while (readChildInfo(&information_type, &cow, &cow_updated, &keys, &progress)) { + updateChildInfo(information_type, cow, cow_updated, keys, progress); } } diff --git a/src/cluster.cpp b/src/cluster.cpp index 72867d9d1..b2391f6b4 100644 --- a/src/cluster.cpp +++ b/src/cluster.cpp @@ -55,7 +55,7 @@ void clusterSendFail(char *nodename); void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request); void clusterUpdateState(void); int clusterNodeGetSlotBit(clusterNode *n, int slot); -sds clusterGenNodesDescription(int filter); +sds clusterGenNodesDescription(int filter, int use_pport); clusterNode *clusterLookupNode(const char *name); int clusterNodeAddSlave(clusterNode *master, clusterNode *slave); int clusterAddSlot(clusterNode *n, int slot); @@ -198,6 +198,9 @@ int clusterLoadConfig(char *filename) { * base port. */ n->cport = busp ? atoi(busp) : n->port + CLUSTER_PORT_INCR; + /* The plaintext port for client in a TLS cluster (n->pport) is not + * stored in nodes.conf. It is received later over the bus protocol. */ + /* Parse flags */ p = s = argv[2]; while(p) { @@ -353,7 +356,7 @@ int clusterSaveConfig(int do_fsync) { /* Get the nodes description and concatenate our "vars" directive to * save currentEpoch and lastVoteEpoch. */ - ci = clusterGenNodesDescription(CLUSTER_NODE_HANDSHAKE); + ci = clusterGenNodesDescription(CLUSTER_NODE_HANDSHAKE, 0); ci = sdscatprintf(ci,"vars currentEpoch %llu lastVoteEpoch %llu\n", (unsigned long long) g_pserver->cluster->currentEpoch, (unsigned long long) g_pserver->cluster->lastVoteEpoch); @@ -372,7 +375,7 @@ int clusterSaveConfig(int do_fsync) { if (write(fd,ci,sdslen(ci)) != (ssize_t)sdslen(ci)) goto err; if (do_fsync) { g_pserver->cluster->todo_before_sleep &= ~CLUSTER_TODO_FSYNC_CONFIG; - fsync(fd); + if (fsync(fd) == -1) goto err; } /* Truncate the file if needed to remove the final \n padding that @@ -452,6 +455,26 @@ int clusterLockConfig(char *filename) { return C_OK; } +/* Derives our ports to be announced in the cluster bus. */ +void deriveAnnouncedPorts(int *announced_port, int *announced_pport, + int *announced_cport) { + int port = g_pserver->tls_cluster ? g_pserver->tls_port : g_pserver->port; + /* Default announced ports. */ + *announced_port = port; + *announced_pport = g_pserver->tls_cluster ? g_pserver->port : 0; + *announced_cport = port + CLUSTER_PORT_INCR; + /* Config overriding announced ports. */ + if (g_pserver->tls_cluster && g_pserver->cluster_announce_tls_port) { + *announced_port = g_pserver->cluster_announce_tls_port; + *announced_pport = g_pserver->cluster_announce_port; + } else if (g_pserver->cluster_announce_port) { + *announced_port = g_pserver->cluster_announce_port; + } + if (g_pserver->cluster_announce_bus_port) { + *announced_cport = g_pserver->cluster_announce_bus_port; + } +} + /* Some flags (currently just the NOFAILOVER flag) may need to be updated * in the "myself" node based on the current configuration of the node, * that may change at runtime via CONFIG SET. This function changes the @@ -546,14 +569,9 @@ void clusterInit(void) { memset(g_pserver->cluster->slots_keys_count,0, sizeof(g_pserver->cluster->slots_keys_count)); - /* Set myself->port / cport to my listening ports, we'll just need to + /* Set myself->port/cport/pport to my listening ports, we'll just need to * discover the IP address via MEET messages. */ - myself->port = port; - myself->cport = port+CLUSTER_PORT_INCR; - if (g_pserver->cluster_announce_port) - myself->port = g_pserver->cluster_announce_port; - if (g_pserver->cluster_announce_bus_port) - myself->cport = g_pserver->cluster_announce_bus_port; + deriveAnnouncedPorts(&myself->port, &myself->pport, &myself->cport); g_pserver->cluster->mf_end = 0; resetManualFailover(); @@ -820,6 +838,7 @@ clusterNode *createClusterNode(char *nodename, int flags) { memset(node->ip,0,sizeof(node->ip)); node->port = 0; node->cport = 0; + node->pport = 0; node->fail_reports = listCreate(); node->voted_time = 0; node->orphaned_time = 0; @@ -1526,6 +1545,7 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) { if (node->link) freeClusterLink(node->link); memcpy(node->ip,g->ip,NET_IP_STR_LEN); node->port = ntohs(g->port); + node->pport = ntohs(g->pport); node->cport = ntohs(g->cport); node->flags &= ~CLUSTER_NODE_NOADDR; } @@ -1547,6 +1567,7 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) { node = createClusterNode(g->nodename, flags); memcpy(node->ip,g->ip,NET_IP_STR_LEN); node->port = ntohs(g->port); + node->pport = ntohs(g->pport); node->cport = ntohs(g->cport); clusterAddNode(node); } @@ -1586,6 +1607,7 @@ int nodeUpdateAddressIfNeeded(clusterNode *node, clusterLink *link, { char ip[NET_IP_STR_LEN] = {0}; int port = ntohs(hdr->port); + int pport = ntohs(hdr->pport); int cport = ntohs(hdr->cport); /* We don't proceed if the link is the same as the sender link, as this @@ -1597,12 +1619,13 @@ int nodeUpdateAddressIfNeeded(clusterNode *node, clusterLink *link, if (link == node->link) return 0; nodeIp2String(ip,link,hdr->myip); - if (node->port == port && node->cport == cport && + if (node->port == port && node->cport == cport && node->pport == pport && strcmp(ip,node->ip) == 0) return 0; /* IP / port is different, update it. */ memcpy(node->ip,ip,sizeof(ip)); node->port = port; + node->pport = pport; node->cport = cport; if (node->link) freeClusterLink(node->link); node->flags &= ~CLUSTER_NODE_NOADDR; @@ -1652,7 +1675,7 @@ void clusterSetNodeAsMaster(clusterNode *n) { * case we receive the info via an UPDATE packet. */ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoch, unsigned char *slots) { int j; - clusterNode *curmaster, *newmaster = NULL; + clusterNode *curmaster = NULL, *newmaster = NULL; /* The dirty slots list is a list of slots for which we lose the ownership * while having still keys inside. This usually happens after a failover * or after a manual cluster reconfiguration operated by the admin. @@ -1663,6 +1686,12 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc uint16_t dirty_slots[CLUSTER_SLOTS]; int dirty_slots_count = 0; + /* We should detect if sender is new master of our shard. + * We will know it if all our slots were migrated to sender, and sender + * has no slots except ours */ + int sender_slots = 0; + int migrated_our_slots = 0; + /* Here we set curmaster to this node or the node this node * replicates to if it's a slave. In the for loop we are * interested to check if slots are taken away from curmaster. */ @@ -1675,6 +1704,8 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc for (j = 0; j < CLUSTER_SLOTS; j++) { if (bitmapTestBit(slots,j)) { + sender_slots++; + /* The slot is already bound to the sender of this message. */ if (g_pserver->cluster->slots[j] == sender) continue; @@ -1701,8 +1732,10 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc dirty_slots_count++; } - if (g_pserver->cluster->slots[j] == curmaster) + if (g_pserver->cluster->slots[j] == curmaster) { newmaster = sender; + migrated_our_slots++; + } clusterDelSlot(j); clusterAddSlot(sender,j); clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG| @@ -1725,7 +1758,9 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc * master. * 2) We are a slave and our master is left without slots. We need * to replicate to the new slots owner. */ - if (newmaster && curmaster->numslots == 0) { + if (newmaster && curmaster->numslots == 0 && + (g_pserver->cluster_allow_replica_migration || + sender_slots == migrated_our_slots)) { serverLog(LL_WARNING, "Configuration change detected. Reconfiguring myself " "as a replica of %.40s", sender->name); @@ -1853,7 +1888,7 @@ int clusterProcessPacket(clusterLink *link) { nodeIsSlave(myself) && myself->slaveof == sender && hdr->mflags[0] & CLUSTERMSG_FLAG0_PAUSED && - g_pserver->cluster->mf_master_offset == 0) + g_pserver->cluster->mf_master_offset == -1) { g_pserver->cluster->mf_master_offset = sender->repl_offset; clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_MANUALFAILOVER); @@ -1904,6 +1939,7 @@ int clusterProcessPacket(clusterLink *link) { node = createClusterNode(NULL,CLUSTER_NODE_HANDSHAKE); nodeIp2String(node->ip,link,hdr->myip); node->port = ntohs(hdr->port); + node->pport = ntohs(hdr->pport); node->cport = ntohs(hdr->cport); clusterAddNode(node); clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG); @@ -1966,6 +2002,7 @@ int clusterProcessPacket(clusterLink *link) { link->node->flags |= CLUSTER_NODE_NOADDR; link->node->ip[0] = '\0'; link->node->port = 0; + link->node->pport = 0; link->node->cport = 0; freeClusterLink(link); clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG); @@ -2473,19 +2510,16 @@ void clusterBuildMessageHdr(clusterMsg *hdr, int type) { hdr->myip[NET_IP_STR_LEN-1] = '\0'; } - /* Handle cluster-announce-port as well. */ - int port = g_pserver->tls_cluster ? g_pserver->tls_port : g_pserver->port; - int announced_port = g_pserver->cluster_announce_port ? - g_pserver->cluster_announce_port : port; - int announced_cport = g_pserver->cluster_announce_bus_port ? - g_pserver->cluster_announce_bus_port : - (port + CLUSTER_PORT_INCR); + /* Handle cluster-announce-[tls-|bus-]port. */ + int announced_port, announced_pport, announced_cport; + deriveAnnouncedPorts(&announced_port, &announced_pport, &announced_cport); memcpy(hdr->myslots,master->slots,sizeof(hdr->myslots)); memset(hdr->slaveof,0,CLUSTER_NAMELEN); if (myself->slaveof != NULL) memcpy(hdr->slaveof,myself->slaveof->name, CLUSTER_NAMELEN); hdr->port = htons(announced_port); + hdr->pport = htons(announced_pport); hdr->cport = htons(announced_cport); hdr->flags = htons(myself->flags); hdr->state = g_pserver->cluster->state; @@ -2542,6 +2576,7 @@ void clusterSetGossipEntry(clusterMsg *hdr, int i, clusterNode *n) { gossip->port = htons(n->port); gossip->cport = htons(n->cport); gossip->flags = htons(n->flags); + gossip->pport = htons(n->pport); gossip->notused1 = 0; } @@ -3140,7 +3175,7 @@ void clusterFailoverReplaceYourMaster(void) { /* This function is called if we are a slave node and our master serving * a non-zero amount of hash slots is in FAIL state. * - * The gaol of this function is: + * The goal of this function is: * 1) To check if we are able to perform a failover, is our data updated? * 2) Try to get elected by masters. * 3) Perform the failover informing all the other nodes. @@ -3185,7 +3220,7 @@ void clusterHandleSlaveFailover(void) { return; } - /* Set data_age to the number of seconds we are disconnected from + /* Set data_age to the number of milliseconds we are disconnected from * the master. */ if (getFirstMaster()->repl_state == REPL_STATE_CONNECTED) { data_age = (mstime_t)(g_pserver->unixtime - getFirstMaster()->master->lastinteraction) @@ -3469,7 +3504,7 @@ void resetManualFailover(void) { g_pserver->cluster->mf_end = 0; /* No manual failover in progress. */ g_pserver->cluster->mf_can_start = 0; g_pserver->cluster->mf_slave = NULL; - g_pserver->cluster->mf_master_offset = 0; + g_pserver->cluster->mf_master_offset = -1; } /* If a manual failover timed out, abort it. */ @@ -3490,7 +3525,7 @@ void clusterHandleManualFailover(void) { * next steps are performed by clusterHandleSlaveFailover(). */ if (g_pserver->cluster->mf_can_start) return; - if (g_pserver->cluster->mf_master_offset == 0) return; /* Wait for offset... */ + if (g_pserver->cluster->mf_master_offset == -1) return; /* Wait for offset... */ if (g_pserver->cluster->mf_master_offset == replicationGetSlaveOffset(getFirstMaster())) { /* Our replication offset matches the master replication offset @@ -3680,7 +3715,6 @@ void clusterCron(void) { now - node->link->ctime > g_pserver->cluster_node_timeout && /* was not already reconnected */ node->ping_sent && /* we already sent a ping */ - node->pong_received < node->ping_sent && /* still waiting pong */ /* and we are waiting for the pong more than timeout/2 */ ping_delay > g_pserver->cluster_node_timeout/2 && /* and in such interval we are not seeing any traffic at all. */ @@ -3763,7 +3797,8 @@ void clusterCron(void) { * the orphaned masters. Note that it does not make sense to try * a migration if there is no master with at least *two* working * slaves. */ - if (orphaned_masters && max_slaves >= 2 && this_slaves == max_slaves) + if (orphaned_masters && max_slaves >= 2 && this_slaves == max_slaves && + g_pserver->cluster_allow_replica_migration) clusterHandleSlaveMigration(max_slaves); } @@ -3873,7 +3908,7 @@ int clusterNodeSetSlotBit(clusterNode *n, int slot) { * However new masters with slots assigned are considered valid * migration targets if the rest of the cluster is not a slave-less. * - * See https://github.com/antirez/redis/issues/3043 for more info. */ + * See https://github.com/redis/redis/issues/3043 for more info. */ if (n->numslots == 1 && clusterMastersHaveSlaves()) n->flags |= CLUSTER_NODE_MIGRATE_TO; } @@ -4181,15 +4216,16 @@ sds representClusterNodeFlags(sds ci, uint16_t flags) { * See clusterGenNodesDescription() top comment for more information. * * The function returns the string representation as an SDS string. */ -sds clusterGenNodeDescription(clusterNode *node) { +sds clusterGenNodeDescription(clusterNode *node, int use_pport) { int j, start; sds ci; + int port = use_pport && node->pport ? node->pport : node->port; /* Node coordinates */ ci = sdscatlen(sdsempty(),node->name,CLUSTER_NAMELEN); ci = sdscatfmt(ci," %s:%i@%i ", node->ip, - node->port, + port, node->cport); /* Flags */ @@ -4300,10 +4336,13 @@ void clusterGenNodesSlotsInfo(int filter) { * include all the known nodes in the representation, including nodes in * the HANDSHAKE state. * + * Setting use_pport to 1 in a TLS cluster makes the result contain the + * plaintext client port rather then the TLS client port of each node. + * * The representation obtained using this function is used for the output * of the CLUSTER NODES function, and as format for the cluster * configuration file (nodes.conf) for a given node. */ -sds clusterGenNodesDescription(int filter) { +sds clusterGenNodesDescription(int filter, int use_pport) { sds ci = sdsempty(), ni; dictIterator *di; dictEntry *de; @@ -4316,7 +4355,7 @@ sds clusterGenNodesDescription(int filter) { clusterNode *node = (clusterNode*)dictGetVal(de); if (node->flags & filter) continue; - ni = clusterGenNodeDescription(node); + ni = clusterGenNodeDescription(node, use_pport); ci = sdscatsds(ci,ni); sdsfree(ni); ci = sdscatlen(ci,"\n",1); @@ -4363,7 +4402,37 @@ int getSlotOrReply(client *c, robj *o) { return (int) slot; } -void clusterReplyMultiBulkSlots(client *c) { +void addNodeReplyForClusterSlot(client *c, clusterNode *node, int start_slot, int end_slot) { + int i, nested_elements = 3; /* slots (2) + master addr (1) */ + void *nested_replylen = addReplyDeferredLen(c); + addReplyLongLong(c, start_slot); + addReplyLongLong(c, end_slot); + addReplyArrayLen(c, 3); + addReplyBulkCString(c, node->ip); + /* Report non-TLS ports to non-TLS client in TLS cluster if available. */ + int use_pport = (g_pserver->tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); + addReplyLongLong(c, use_pport && node->pport ? node->pport : node->port); + addReplyBulkCBuffer(c, node->name, CLUSTER_NAMELEN); + + /* Remaining nodes in reply are replicas for slot range */ + for (i = 0; i < node->numslaves; i++) { + /* This loop is copy/pasted from clusterGenNodeDescription() + * with modifications for per-slot node aggregation. */ + if (nodeFailed(node->slaves[i])) continue; + addReplyArrayLen(c, 3); + addReplyBulkCString(c, node->slaves[i]->ip); + /* Report slave's non-TLS port to non-TLS client in TLS cluster */ + addReplyLongLong(c, (use_pport && node->slaves[i]->pport ? + node->slaves[i]->pport : + node->slaves[i]->port)); + addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN); + nested_elements++; + } + setDeferredArrayLen(c, nested_replylen, nested_elements); +} + +void clusterReplyMultiBulkSlots(client * c) { /* Format: 1) 1) start slot * 2) end slot * 3) 1) master IP @@ -4374,69 +4443,29 @@ void clusterReplyMultiBulkSlots(client *c) { * 3) node ID * ... continued until done */ - - int num_masters = 0; + clusterNode *n = NULL; + int num_masters = 0, start = -1; void *slot_replylen = addReplyDeferredLen(c); - dictEntry *de; - dictIterator *di = dictGetSafeIterator(g_pserver->cluster->nodes); - while((de = dictNext(di)) != NULL) { - clusterNode *node = (clusterNode*)dictGetVal(de); - int j = 0, start = -1; - int i, nested_elements = 0; - - /* Skip slaves (that are iterated when producing the output of their - * master) and masters not serving any slot. */ - if (!nodeIsMaster(node) || node->numslots == 0) continue; - - for(i = 0; i < node->numslaves; i++) { - if (nodeFailed(node->slaves[i])) continue; - nested_elements++; + for (int i = 0; i <= CLUSTER_SLOTS; i++) { + /* Find start node and slot id. */ + if (n == NULL) { + if (i == CLUSTER_SLOTS) break; + n = g_pserver->cluster->slots[i]; + start = i; + continue; } - for (j = 0; j < CLUSTER_SLOTS; j++) { - int bit, i; - - if ((bit = clusterNodeGetSlotBit(node,j)) != 0) { - if (start == -1) start = j; - } - if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) { - addReplyArrayLen(c, nested_elements + 3); /* slots (2) + master addr (1). */ - - if (bit && j == CLUSTER_SLOTS-1) j++; - - /* If slot exists in output map, add to it's list. - * else, create a new output map for this slot */ - if (start == j-1) { - addReplyLongLong(c, start); /* only one slot; low==high */ - addReplyLongLong(c, start); - } else { - addReplyLongLong(c, start); /* low */ - addReplyLongLong(c, j-1); /* high */ - } - start = -1; - - /* First node reply position is always the master */ - addReplyArrayLen(c, 3); - addReplyBulkCString(c, node->ip); - addReplyLongLong(c, node->port); - addReplyBulkCBuffer(c, node->name, CLUSTER_NAMELEN); - - /* Remaining nodes in reply are replicas for slot range */ - for (i = 0; i < node->numslaves; i++) { - /* This loop is copy/pasted from clusterGenNodeDescription() - * with modifications for per-slot node aggregation */ - if (nodeFailed(node->slaves[i])) continue; - addReplyArrayLen(c, 3); - addReplyBulkCString(c, node->slaves[i]->ip); - addReplyLongLong(c, node->slaves[i]->port); - addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN); - } - num_masters++; - } + /* Add cluster slots info when occur different node with start + * or end of slot. */ + if (i == CLUSTER_SLOTS || n != g_pserver->cluster->slots[i]) { + addNodeReplyForClusterSlot(c, n, start, i-1); + num_masters++; + if (i == CLUSTER_SLOTS) break; + n = g_pserver->cluster->slots[i]; + start = i; } } - dictReleaseIterator(di); setDeferredArrayLen(c, slot_replylen, num_masters); } @@ -4525,7 +4554,11 @@ NULL } } else if (!strcasecmp(szFromObj(c->argv[1]),"nodes") && c->argc == 2) { /* CLUSTER NODES */ - sds nodes = clusterGenNodesDescription(0); + /* Report plaintext ports, only if cluster is TLS but client is known to + * be non-TLS). */ + int use_pport = (g_pserver->tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); + sds nodes = clusterGenNodesDescription(0, use_pport); addReplyVerbatim(c,nodes,sdslen(nodes),"txt"); sdsfree(nodes); } else if (!strcasecmp(szFromObj(c->argv[1]),"myid") && c->argc == 2) { @@ -4901,9 +4934,12 @@ NULL return; } + /* Use plaintext port if cluster is TLS but client is non-TLS. */ + int use_pport = (g_pserver->tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); addReplyArrayLen(c,n->numslaves); for (j = 0; j < n->numslaves; j++) { - sds ni = clusterGenNodeDescription(n->slaves[j]); + sds ni = clusterGenNodeDescription(n->slaves[j], use_pport); addReplyBulkCString(c,ni); sdsfree(ni); } @@ -5930,18 +5966,14 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in * cluster is down. */ if (error_code) *error_code = CLUSTER_REDIR_DOWN_STATE; return NULL; - } else if ((cmd->flags & CMD_WRITE) && !(cmd->proc == evalCommand) - && !(cmd->proc == evalShaCommand)) - { - /* The cluster is configured to allow read only commands - * but this command is neither readonly, nor EVAL or - * EVALSHA. */ + } else if (cmd->flags & CMD_WRITE) { + /* The cluster is configured to allow read only commands */ if (error_code) *error_code = CLUSTER_REDIR_DOWN_RO_STATE; return NULL; } else { /* Fall through and allow the command to be executed: - * this happens when g_pserver->cluster_allow_reads_when_down is - * true and the command is a readonly command or EVAL / EVALSHA. */ + * this happens when server.cluster_allow_reads_when_down is + * true and the command is not a write command */ } } @@ -5982,7 +6014,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in int is_write_command = (c->cmd->flags & CMD_WRITE) || (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_WRITE)); if (c->flags & CLIENT_READONLY && - (!is_write_command || cmd->proc == evalCommand || cmd->proc == evalShaCommand) && + !is_write_command && nodeIsSlave(myself) && myself->slaveof == n) { @@ -6019,10 +6051,15 @@ void clusterRedirectClient(client *c, clusterNode *n, int hashslot, int error_co } else if (error_code == CLUSTER_REDIR_MOVED || error_code == CLUSTER_REDIR_ASK) { + /* Redirect to IP:port. Include plaintext port if cluster is TLS but + * client is non-TLS. */ + int use_pport = (g_pserver->tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); + int port = use_pport && n->pport ? n->pport : n->port; addReplyErrorSds(c,sdscatprintf(sdsempty(), "-%s %d %s:%d", (error_code == CLUSTER_REDIR_ASK) ? "ASK" : "MOVED", - hashslot,n->ip,n->port)); + hashslot, n->ip, port)); } else { serverPanic("getNodeByQuery() unknown error."); } diff --git a/src/cluster.h b/src/cluster.h index 9d370b360..db82c3f24 100644 --- a/src/cluster.h +++ b/src/cluster.h @@ -139,7 +139,9 @@ typedef struct clusterNode { mstime_t orphaned_time; /* Starting time of orphaned master condition */ long long repl_offset; /* Last known repl offset for this node. */ char ip[NET_IP_STR_LEN]; /* Latest known IP address of this node */ - int port; /* Latest known clients port of this node */ + int port; /* Latest known clients port (TLS or plain). */ + int pport; /* Latest known clients plaintext port. Only used + if the main clients port is for TLS. */ int cport; /* Latest known cluster port of this node. */ clusterLink *link; /* TCP/IP link with this node */ list *fail_reports; /* List of nodes signaling this as failing */ @@ -172,7 +174,7 @@ typedef struct clusterState { clusterNode *mf_slave; /* Slave performing the manual failover. */ /* Manual failover state of slave. */ long long mf_master_offset; /* Master offset the slave needs to start MF - or zero if still not received. */ + or -1 if still not received. */ int mf_can_start; /* If non-zero signal that the manual failover can start requesting masters vote. */ /* The following fields are used by masters to take state on elections. */ @@ -198,7 +200,8 @@ typedef struct { uint16_t port; /* base port last time it was seen */ uint16_t cport; /* cluster port last time it was seen */ uint16_t flags; /* node->flags copy */ - uint32_t notused1; + uint16_t pport; /* plaintext-port, when base port is TLS */ + uint16_t notused1; } clusterMsgDataGossip; typedef struct { @@ -271,7 +274,8 @@ typedef struct { unsigned char myslots[CLUSTER_SLOTS/8]; char slaveof[CLUSTER_NAMELEN]; char myip[NET_IP_STR_LEN]; /* Sender IP, if not all zeroed. */ - char notused1[34]; /* 34 bytes reserved for future usage. */ + char notused1[32]; /* 32 bytes reserved for future usage. */ + uint16_t pport; /* Sender TCP plaintext port, if base port is TLS */ uint16_t cport; /* Sender TCP cluster bus port */ uint16_t flags; /* Sender node flags */ unsigned char state; /* Cluster state from the POV of the sender */ diff --git a/src/config.cpp b/src/config.cpp index 035162b73..41108c79a 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -231,8 +231,6 @@ typedef union typeData { typedef struct typeInterface { /* Called on server start, to init the server with default value */ void (*init)(typeData data); - /* Called on server start, should return 1 on success, 0 on error and should set err */ - int (*load)(typeData data, sds *argc, int argv, const char **err); /* Called on server startup and CONFIG SET, returns 1 on success, 0 on error * and can set a verbose err string, update is true when called from CONFIG SET */ int (*set)(typeData data, sds value, int update, const char **err); @@ -245,11 +243,16 @@ typedef struct typeInterface { typedef struct standardConfig { const char *name; /* The user visible name of this config */ const char *alias; /* An alias that can also be used for this config */ - const int modifiable; /* Can this value be updated by CONFIG SET? */ + const unsigned int flags; /* Flags for this specific config */ typeInterface interface; /* The function pointers that define the type interface */ typeData data; /* The type specific data exposed used by the interface */ } standardConfig; +#define MODIFIABLE_CONFIG 0 /* This is the implied default for a standard + * config, which is mutable. */ +#define IMMUTABLE_CONFIG (1ULL<<0) /* Can this value only be set at startup? */ +#define SENSITIVE_CONFIG (1ULL<<1) /* Does this value contain sensitive information */ + extern standardConfig configs[]; /*----------------------------------------------------------------------------- @@ -692,6 +695,10 @@ void loadServerConfigFromString(char *config) { goto loaderr; } + /* To ensure backward compatibility and work while hz is out of range */ + if (g_pserver->config_hz < CONFIG_MIN_HZ) g_pserver->config_hz = CONFIG_MIN_HZ; + if (g_pserver->config_hz > CONFIG_MAX_HZ) g_pserver->config_hz = CONFIG_MAX_HZ; + sdsfreesplitres(lines,totlines); return; @@ -788,9 +795,13 @@ void configSetCommand(client *c) { /* Iterate the configs that are standard */ for (standardConfig *config = configs; config->name != NULL; config++) { - if(config->modifiable && (!strcasecmp(szFromObj(c->argv[2]),config->name) || + if (!(config->flags & IMMUTABLE_CONFIG) && + (!strcasecmp(szFromObj(c->argv[2]),config->name) || (config->alias && !strcasecmp(szFromObj(c->argv[2]),config->alias)))) { + if (config->flags & SENSITIVE_CONFIG) { + preventCommandLogging(c); + } if (!config->interface.set(config->data,szFromObj(o),1,&errstr)) { goto badfmt; } @@ -1482,15 +1493,20 @@ void rewriteConfigSaveOption(struct rewriteConfigState *state) { return; } - /* Note that if there are no save parameters at all, all the current - * config line with "save" will be detected as orphaned and deleted, - * resulting into no RDB persistence as expected. */ - for (j = 0; j < g_pserver->saveparamslen; j++) { - line = sdscatprintf(sdsempty(),"save %ld %d", - (long) g_pserver->saveparams[j].seconds, g_pserver->saveparams[j].changes); - rewriteConfigRewriteLine(state,"save",line,1); + /* Rewrite save parameters, or an empty 'save ""' line to avoid the + * defaults from being used. + */ + if (!g_pserver->saveparamslen) { + rewriteConfigRewriteLine(state,"save",sdsnew("save \"\""),1); + } else { + for (j = 0; j < g_pserver->saveparamslen; j++) { + line = sdscatprintf(sdsempty(),"save %ld %d", + (long) g_pserver->saveparams[j].seconds, g_pserver->saveparams[j].changes); + rewriteConfigRewriteLine(state,"save",line,1); + } } - /* Mark "save" as processed in case g_pserver->saveparamslen is zero. */ + + /* Mark "save" as processed in case server.saveparamslen is zero. */ rewriteConfigMarkAsProcessed(state,"save"); } @@ -1833,14 +1849,11 @@ int rewriteConfig(char *path, int force_all) { #define LOADBUF_SIZE 256 static char loadbuf[LOADBUF_SIZE]; -#define MODIFIABLE_CONFIG 1 -#define IMMUTABLE_CONFIG 0 - -#define embedCommonConfig(config_name, config_alias, is_modifiable) \ - config_name, config_alias, is_modifiable, +#define embedCommonConfig(config_name, config_alias, config_flags) \ + config_name, config_alias, config_flags, #define embedConfigInterface(initfn, setfn, getfn, rewritefn) { \ - initfn, nullptr, setfn, getfn, rewritefn, \ + initfn, setfn, getfn, rewritefn, \ }, /* What follows is the generic config types that are supported. To add a new @@ -1887,11 +1900,11 @@ static void boolConfigRewrite(typeData data, const char *name, struct rewriteCon rewriteConfigYesNoOption(state, name,*(data.yesno.config), data.yesno.default_value); } -constexpr standardConfig createBoolConfig(const char *name, const char *alias, int modifiable, int &config_addr, int defaultValue, int (*is_valid)(int val, const char **err), int (*update)(int val, int prev, const char **err)) +constexpr standardConfig createBoolConfig(const char *name, const char *alias, unsigned flags, int &config_addr, int defaultValue, int (*is_valid)(int val, const char **err), int (*update)(int val, int prev, const char **err)) { standardConfig conf = { - embedCommonConfig(name, alias, modifiable) - { boolConfigInit, nullptr, boolConfigSet, boolConfigGet, boolConfigRewrite } + embedCommonConfig(name, alias, flags) + { boolConfigInit, boolConfigSet, boolConfigGet, boolConfigRewrite } }; conf.data.yesno.config = &config_addr; conf.data.yesno.default_value = defaultValue; @@ -1962,9 +1975,9 @@ static void sdsConfigRewrite(typeData data, const char *name, struct rewriteConf #define ALLOW_EMPTY_STRING 0 #define EMPTY_STRING_IS_NULL 1 -constexpr standardConfig createStringConfig(const char *name, const char *alias, int modifiable, int empty_to_null, char *&config_addr, const char *defaultValue, int (*is_valid)(char*,const char**), int (*update)(char*,char*,const char**)) { +constexpr standardConfig createStringConfig(const char *name, const char *alias, unsigned flags, int empty_to_null, char *&config_addr, const char *defaultValue, int (*is_valid)(char*,const char**), int (*update)(char*,char*,const char**)) { standardConfig conf = { - embedCommonConfig(name, alias, modifiable) + embedCommonConfig(name, alias, flags) embedConfigInterface(stringConfigInit, stringConfigSet, stringConfigGet, stringConfigRewrite) }; conf.data.string = { @@ -1977,9 +1990,9 @@ constexpr standardConfig createStringConfig(const char *name, const char *alias, return conf; } -constexpr standardConfig createSDSConfig(const char *name, const char *alias, int modifiable, int empty_to_null, sds &config_addr, const char *defaultValue, int (*is_valid)(char*,const char**), int (*update)(char*,char*,const char**)) { +constexpr standardConfig createSDSConfig(const char *name, const char *alias, unsigned flags, int empty_to_null, sds &config_addr, const char *defaultValue, int (*is_valid)(char*,const char**), int (*update)(char*,char*,const char**)) { standardConfig conf = { - embedCommonConfig(name, alias, modifiable) + embedCommonConfig(name, alias, flags) embedConfigInterface(sdsConfigInit, sdsConfigSet, sdsConfigGet, sdsConfigRewrite) }; conf.data.sds = { @@ -2036,9 +2049,9 @@ static void enumConfigRewrite(typeData data, const char *name, struct rewriteCon rewriteConfigEnumOption(state, name,*(data.enumd.config), data.enumd.enum_value, data.enumd.default_value); } -constexpr standardConfig createEnumConfig(const char *name, const char *alias, int modifiable, configEnum *enumVal, int &config_addr, int defaultValue, int (*is_valid)(int,const char**), int (*update)(int,int,const char**)) { +constexpr standardConfig createEnumConfig(const char *name, const char *alias, unsigned flags, configEnum *enumVal, int &config_addr, int defaultValue, int (*is_valid)(int,const char**), int (*update)(int,int,const char**)) { standardConfig c = { - embedCommonConfig(name, alias, modifiable) + embedCommonConfig(name, alias, flags) embedConfigInterface(enumConfigInit, enumConfigSet, enumConfigGet, enumConfigRewrite) }; c.data.enumd = { @@ -2194,9 +2207,9 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite #define INTEGER_CONFIG 0 #define MEMORY_CONFIG 1 -constexpr standardConfig embedCommonNumericalConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { +constexpr standardConfig embedCommonNumericalConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { standardConfig conf = { - embedCommonConfig(name, alias, modifiable) + embedCommonConfig(name, alias, flags) embedConfigInterface(numericConfigInit, numericConfigSet, numericConfigGet, numericConfigRewrite) }; conf.data.numeric.is_memory = (memory); @@ -2208,73 +2221,73 @@ constexpr standardConfig embedCommonNumericalConfig(const char *name, const char return conf; } -constexpr standardConfig createIntConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, int &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) +constexpr standardConfig createIntConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, int &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - standardConfig conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); + standardConfig conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_INT; conf.data.numeric.config.i = &config_addr; return conf; } -constexpr standardConfig createUIntConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, unsigned int &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) +constexpr standardConfig createUIntConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, unsigned int &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_UINT; conf.data.numeric.config.ui = &(config_addr); return conf; } -constexpr standardConfig createLongConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createLongConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_LONG; conf.data.numeric.config.l = &(config_addr); return conf; } -constexpr standardConfig createULongConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, unsigned long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createULongConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, unsigned long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_ULONG; conf.data.numeric.config.ul = &(config_addr); return conf; } -constexpr standardConfig createLongLongConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, long long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createLongLongConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, long long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_LONG_LONG; conf.data.numeric.config.ll = &(config_addr); return conf; } -constexpr standardConfig createULongLongConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, unsigned long long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createULongLongConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, unsigned long long &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_ULONG_LONG; conf.data.numeric.config.ull = &(config_addr); return conf; } -constexpr standardConfig createSizeTConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, size_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createSizeTConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, size_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_SIZE_T; conf.data.numeric.config.st = &(config_addr); return conf; } -constexpr standardConfig createSSizeTConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, ssize_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createSSizeTConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, ssize_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_SSIZE_T; conf.data.numeric.config.sst = &(config_addr); return conf; } -constexpr standardConfig createTimeTConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, time_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createTimeTConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, time_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_TIME_T; conf.data.numeric.config.tt = &(config_addr); return conf; } -constexpr standardConfig createOffTConfig(const char *name, const char *alias, int modifiable, long long lower, long long upper, off_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { - auto conf = embedCommonNumericalConfig(name, alias, modifiable, lower, upper, defaultValue, memory, is_valid, update); +constexpr standardConfig createOffTConfig(const char *name, const char *alias, unsigned flags, long long lower, long long upper, off_t &config_addr, long long defaultValue, int memory, int (*is_valid)(long long, const char**), int (*update)(long long, long long, const char**)) { + auto conf = embedCommonNumericalConfig(name, alias, flags, lower, upper, defaultValue, memory, is_valid, update); conf.data.numeric.numeric_type = NUMERIC_TYPE_OFF_T; conf.data.numeric.config.ot = &(config_addr); return conf; @@ -2614,13 +2627,15 @@ standardConfig configs[] = { createBoolConfig("crash-memcheck-enabled", NULL, MODIFIABLE_CONFIG, g_pserver->memcheck_enabled, 1, NULL, NULL), createBoolConfig("use-exit-on-panic", NULL, MODIFIABLE_CONFIG, g_pserver->use_exit_on_panic, 0, NULL, NULL), createBoolConfig("disable-thp", NULL, MODIFIABLE_CONFIG, g_pserver->disable_thp, 1, NULL, NULL), + createBoolConfig("cluster-allow-replica-migration", NULL, MODIFIABLE_CONFIG, g_pserver->cluster_allow_replica_migration, 1, NULL, NULL), + createBoolConfig("replica-announced", NULL, MODIFIABLE_CONFIG, g_pserver->replica_announced, 1, NULL, NULL), /* String Configs */ createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->acl_filename, "", NULL, NULL), createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->unixsocket, NULL, NULL, NULL), createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.pidfile, NULL, NULL, NULL), createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->slave_announce_ip, NULL, NULL, NULL), - createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.default_masteruser, NULL, NULL, updateMasterAuthConfig), + createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, cserver.default_masteruser, NULL, NULL, updateMasterAuthConfig), createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->cluster_announce_ip, NULL, NULL, NULL), createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->syslog_ident, "redis", NULL, NULL), createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->rdb_filename, CONFIG_DEFAULT_RDB_FILENAME, isValidDBfilename, NULL), @@ -2633,8 +2648,8 @@ standardConfig configs[] = { createStringConfig("proc-title-template", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, cserver.proc_title_template, CONFIG_DEFAULT_PROC_TITLE_TEMPLATE, isValidProcTitleTemplate, updateProcTitleTemplate), /* SDS Configs */ - createSDSConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.default_masterauth, NULL, NULL, updateMasterAuthConfig), - createSDSConfig("requirepass", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->requirepass, NULL, NULL, updateRequirePass), + createSDSConfig("masterauth", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, cserver.default_masterauth, NULL, NULL, updateMasterAuthConfig), + createSDSConfig("requirepass", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->requirepass, NULL, NULL, updateRequirePass), /* Enum Configs */ createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, cserver.supervised_mode, SUPERVISED_NONE, NULL, NULL), @@ -2644,7 +2659,7 @@ standardConfig configs[] = { createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, g_pserver->maxmemory_policy, MAXMEMORY_NO_EVICTION, NULL, NULL), createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, g_pserver->aof_fsync, AOF_FSYNC_EVERYSEC, NULL, NULL), createEnumConfig("oom-score-adj", NULL, MODIFIABLE_CONFIG, oom_score_adj_enum, g_pserver->oom_score_adj, OOM_SCORE_ADJ_NO, NULL, updateOOMScoreAdj), - createEnumConfig("acl-pubsub-default", NULL, MODIFIABLE_CONFIG, acl_pubsub_default_enum, g_pserver->acl_pubusub_default, USER_FLAG_ALLCHANNELS, NULL, NULL), + createEnumConfig("acl-pubsub-default", NULL, MODIFIABLE_CONFIG, acl_pubsub_default_enum, g_pserver->acl_pubsub_default, USER_FLAG_ALLCHANNELS, NULL, NULL), createEnumConfig("sanitize-dump-payload", NULL, MODIFIABLE_CONFIG, sanitize_dump_payload_enum, cserver.sanitize_dump_payload, SANITIZE_DUMP_NO, NULL, NULL), /* Integer configs */ @@ -2670,6 +2685,7 @@ standardConfig configs[] = { createIntConfig("tcp-backlog", NULL, IMMUTABLE_CONFIG, 0, INT_MAX, g_pserver->tcp_backlog, 511, INTEGER_CONFIG, NULL, NULL), /* TCP listen backlog. */ createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->cluster_announce_bus_port, 0, INTEGER_CONFIG, NULL, NULL), /* Default: Use +10000 offset. */ createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->cluster_announce_port, 0, INTEGER_CONFIG, NULL, NULL), /* Use g_pserver->port */ + createIntConfig("cluster-announce-tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->cluster_announce_tls_port, 0, INTEGER_CONFIG, NULL, NULL), /* Use server.tls_port */ createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->repl_timeout, 60, INTEGER_CONFIG, NULL, NULL), createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->repl_ping_slave_period, 10, INTEGER_CONFIG, NULL, NULL), createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL), @@ -2734,8 +2750,10 @@ standardConfig configs[] = { createBoolConfig("tls-session-caching", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.session_caching, 1, NULL, updateTlsCfgBool), createStringConfig("tls-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.cert_file, NULL, NULL, updateTlsCfg), createStringConfig("tls-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.key_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-key-file-pass", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.key_file_pass, NULL, NULL, updateTlsCfg), createStringConfig("tls-client-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_cert_file, NULL, NULL, updateTlsCfg), createStringConfig("tls-client-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_key_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-client-key-file-pass", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_key_file_pass, NULL, NULL, updateTlsCfg), createStringConfig("tls-dh-params-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.dh_params_file, NULL, NULL, updateTlsCfg), createStringConfig("tls-ca-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_file, NULL, NULL, updateTlsCfg), createStringConfig("tls-ca-cert-dir", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_dir, NULL, NULL, updateTlsCfg), diff --git a/src/crc64.c b/src/crc64.c index 6c9432c4a..d4db4158e 100644 --- a/src/crc64.c +++ b/src/crc64.c @@ -127,9 +127,10 @@ uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { #include #define UNUSED(x) (void)(x) -int crc64Test(int argc, char *argv[]) { +int crc64Test(int argc, char *argv[], int accurate) { UNUSED(argc); UNUSED(argv); + UNUSED(accurate); crc64_init(); printf("[calcula]: e9c6d914c4b8d9ca == %016" PRIx64 "\n", (uint64_t)_crc64(0, "123456789", 9)); diff --git a/src/crc64.h b/src/crc64.h index 08ac9f7b2..a0bf15b99 100644 --- a/src/crc64.h +++ b/src/crc64.h @@ -11,7 +11,7 @@ void crc64_init(void); uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); #ifdef REDIS_TEST -int crc64Test(int argc, char *argv[]); +int crc64Test(int argc, char *argv[], int accurate); #endif #ifdef __cplusplus diff --git a/src/db.cpp b/src/db.cpp index 498aff0aa..0eccc5d0d 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -168,9 +168,9 @@ robj_roptr lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) { keymiss: if (!(flags & LOOKUP_NONOTIFY)) { - g_pserver->stat_keyspace_misses++; notifyKeyspaceEvent(NOTIFY_KEY_MISS, "keymiss", key, db->id); } + g_pserver->stat_keyspace_misses++; return NULL; } @@ -194,16 +194,24 @@ robj *lookupKeyWriteWithFlags(redisDb *db, robj *key, int flags) { robj *lookupKeyWrite(redisDb *db, robj *key) { return lookupKeyWriteWithFlags(db, key, LOOKUP_NONE); } - +static void SentReplyOnKeyMiss(client *c, robj *reply){ + serverAssert(sdsEncodedObject(reply)); + sds rep = szFromObj(reply); + if (sdslen(rep) > 1 && rep[0] == '-'){ + addReplyErrorObject(c, reply); + } else { + addReply(c,reply); + } +} robj_roptr lookupKeyReadOrReply(client *c, robj *key, robj *reply) { robj_roptr o = lookupKeyRead(c->db, key); - if (!o) addReply(c,reply); + if (!o) SentReplyOnKeyMiss(c, reply); return o; } robj *lookupKeyWriteOrReply(client *c, robj *key, robj *reply) { robj *o = lookupKeyWrite(c->db, key); - if (!o) addReply(c,reply); + if (!o) SentReplyOnKeyMiss(c, reply); return o; } @@ -1681,8 +1689,13 @@ void propagateExpire(redisDb *db, robj *key, int lazy) { incrRefCount(argv[0]); incrRefCount(argv[1]); + /* If the master decided to expire a key we must propagate it to replicas no matter what.. + * Even if module executed a command without asking for propagation. */ + int prev_replication_allowed = g_pserver->replication_allowed; + g_pserver->replication_allowed = 1; if (!g_pserver->fActiveReplica) // Active replicas do their own expiries, do not propogate propagate(cserver.delCommand,db->id,argv,2,PROPAGATE_AOF|PROPAGATE_REPL); + g_pserver->replication_allowed = prev_replication_allowed; decrRefCount(argv[0]); decrRefCount(argv[1]); diff --git a/src/debug.cpp b/src/debug.cpp index 9bc2d3404..021cf9c4a 100644 --- a/src/debug.cpp +++ b/src/debug.cpp @@ -259,7 +259,7 @@ void xorObjectDigest(redisDb *db, robj_roptr keyobj, unsigned char *digest, robj } streamIteratorStop(&si); } else if (o->type == OBJ_MODULE) { - RedisModuleDigest md; + RedisModuleDigest md = {{0},{0}}; moduleValue *mv = (moduleValue*)ptrFromObj(o); moduleType *mt = mv->type; moduleInitDigestContext(md); @@ -455,7 +455,7 @@ void debugCommand(client *c) { " conflicting keys will generate an exception and kill the server." " * NOSAVE: the database will be loaded from an existing RDB file.", " Examples:", -" * DEBUG RELOAD: verify that the server is able to persist, flsuh and reload", +" * DEBUG RELOAD: verify that the server is able to persist, flush and reload", " the database.", " * DEBUG RELOAD NOSAVE: replace the current database with the contents of an", " existing RDB file.", @@ -487,7 +487,7 @@ NULL } else if (!strcasecmp(szFromObj(c->argv[1]),"segfault")) { *((char*)-1) = 'x'; } else if (!strcasecmp(szFromObj(c->argv[1]),"panic")) { - serverPanic("DEBUG PANIC called at Unix time %ld", time(NULL)); + serverPanic("DEBUG PANIC called at Unix time %lld", (long long)time(NULL)); } else if (!strcasecmp(szFromObj(c->argv[1]),"restart") || !strcasecmp(szFromObj(c->argv[1]),"crash-and-recover")) { @@ -941,6 +941,7 @@ NULL /* =========================== Crash handling ============================== */ void _serverAssert(const char *estr, const char *file, int line) { + g_fInCrash = true; bugReportStart(); serverLog(LL_WARNING,"=== ASSERTION FAILED ==="); serverLog(LL_WARNING,"==> %s:%d '%s' is not true",file,line,estr); @@ -1030,12 +1031,14 @@ void _serverAssertWithInfo(const client *c, robj_roptr o, const char *estr, cons } void _serverPanic(const char *file, int line, const char *msg, ...) { + g_fInCrash = true; va_list ap; va_start(ap,msg); char fmtmsg[256]; vsnprintf(fmtmsg,sizeof(fmtmsg),msg,ap); va_end(ap); + g_fInCrash = true; bugReportStart(); serverLog(LL_WARNING,"------------------------------------------------"); serverLog(LL_WARNING,"!!! Software Failure. Press left mouse button to continue"); @@ -1916,7 +1919,7 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { serverLog(LL_WARNING, "Accessing address: %p", (void*)info->si_addr); } - if (info->si_pid != -1) { + if (info->si_code <= SI_USER && info->si_pid != -1) { serverLog(LL_WARNING, "Killed by PID: %ld, UID: %d", (long) info->si_pid, info->si_uid); } @@ -1944,6 +1947,8 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { } void printCrashReport(void) { + g_fInCrash = true; + /* Log INFO and CLIENT LIST */ logServerInfo(); diff --git a/src/defrag.cpp b/src/defrag.cpp index 25c7f8337..b693e29bb 100644 --- a/src/defrag.cpp +++ b/src/defrag.cpp @@ -371,7 +371,9 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) { if ((newsds = activeDefragSds(sdsele))) { /* When defragging an sds value, we need to update the dict key */ uint64_t hash = dictGetHash(d, newsds); - replaceSatelliteDictKeyPtrAndOrDefragDictEntry(d, sdsele, newsds, hash, &defragged); + dictEntry **deref = dictFindEntryRefByPtrAndHash(d, sdsele, hash); + if (deref) + (*deref)->key = newsds; ln->value = newsds; defragged++; } diff --git a/src/dict.cpp b/src/dict.cpp index c7ef75127..fdda6baf6 100644 --- a/src/dict.cpp +++ b/src/dict.cpp @@ -45,11 +45,7 @@ #include "dict.h" #include "zmalloc.h" -#ifndef DICT_BENCHMARK_MAIN #include "redisassert.h" -#else -#include -#endif /* Using dictEnableResize() / dictDisableResize() we make possible to * enable/disable resizing of the hash table as needed. This is very important @@ -1176,20 +1172,18 @@ void dictGetStats(char *buf, size_t bufsize, dict *d) { /* ------------------------------- Benchmark ---------------------------------*/ -#ifdef DICT_BENCHMARK_MAIN - -#include "sds.h" +#ifdef REDIS_TEST uint64_t hashCallback(const void *key) { - return dictGenHashFunction((unsigned char*)key, sdslen((char*)key)); + return dictGenHashFunction((unsigned char*)key, strlen((char*)key)); } int compareCallback(void *privdata, const void *key1, const void *key2) { int l1,l2; DICT_NOTUSED(privdata); - l1 = sdslen((sds)key1); - l2 = sdslen((sds)key2); + l1 = strlen((char*)key1); + l2 = strlen((char*)key2); if (l1 != l2) return 0; return memcmp(key1, key2, l1) == 0; } @@ -1197,7 +1191,19 @@ int compareCallback(void *privdata, const void *key1, const void *key2) { void freeCallback(void *privdata, void *val) { DICT_NOTUSED(privdata); - sdsfree(val); + zfree(val); +} + +char *stringFromLongLong(long long value) { + char buf[32]; + int len; + char *s; + + len = sprintf(buf,"%lld",value); + s = zmalloc(len+1); + memcpy(s, buf, len); + s[len] = '\0'; + return s; } dictType BenchmarkDictType = { @@ -1216,22 +1222,26 @@ dictType BenchmarkDictType = { printf(msg ": %ld items in %lld ms\n", count, elapsed); \ } while(0) -/* dict-benchmark [count] */ -int main(int argc, char **argv) { +/* ./redis-server test dict [ | --accurate] */ +int dictTest(int argc, char **argv, int accurate) { long j; long long start, elapsed; dict *dict = dictCreate(&BenchmarkDictType,NULL); long count = 0; - if (argc == 2) { - count = strtol(argv[1],NULL,10); + if (argc == 4) { + if (accurate) { + count = 5000000; + } else { + count = strtol(argv[3],NULL,10); + } } else { - count = 5000000; + count = 5000; } start_benchmark(); for (j = 0; j < count; j++) { - int retval = dictAdd(dict,sdsfromlonglong(j),(void*)j); + int retval = dictAdd(dict,stringFromLongLong(j),(void*)j); assert(retval == DICT_OK); } end_benchmark("Inserting"); @@ -1244,28 +1254,28 @@ int main(int argc, char **argv) { start_benchmark(); for (j = 0; j < count; j++) { - sds key = sdsfromlonglong(j); + char *key = stringFromLongLong(j); dictEntry *de = dictFind(dict,key); assert(de != NULL); - sdsfree(key); + zfree(key); } end_benchmark("Linear access of existing elements"); start_benchmark(); for (j = 0; j < count; j++) { - sds key = sdsfromlonglong(j); + char *key = stringFromLongLong(j); dictEntry *de = dictFind(dict,key); assert(de != NULL); - sdsfree(key); + zfree(key); } end_benchmark("Linear access of existing elements (2nd round)"); start_benchmark(); for (j = 0; j < count; j++) { - sds key = sdsfromlonglong(rand() % count); + char *key = stringFromLongLong(rand() % count); dictEntry *de = dictFind(dict,key); assert(de != NULL); - sdsfree(key); + zfree(key); } end_benchmark("Random access of existing elements"); @@ -1278,17 +1288,17 @@ int main(int argc, char **argv) { start_benchmark(); for (j = 0; j < count; j++) { - sds key = sdsfromlonglong(rand() % count); + char *key = stringFromLongLong(rand() % count); key[0] = 'X'; dictEntry *de = dictFind(dict,key); assert(de == NULL); - sdsfree(key); + zfree(key); } end_benchmark("Accessing missing"); start_benchmark(); for (j = 0; j < count; j++) { - sds key = sdsfromlonglong(j); + char *key = stringFromLongLong(j); int retval = dictDelete(dict,key); assert(retval == DICT_OK); key[0] += 17; /* Change first number to letter. */ @@ -1296,5 +1306,7 @@ int main(int argc, char **argv) { assert(retval == DICT_OK); } end_benchmark("Removing and adding"); + dictRelease(dict); + return 0; } #endif diff --git a/src/dict.h b/src/dict.h index 51596e49f..b07f63a32 100644 --- a/src/dict.h +++ b/src/dict.h @@ -238,6 +238,10 @@ extern dictType dictTypeHeapStringCopyKey; extern dictType dictTypeHeapStrings; extern dictType dictTypeHeapStringCopyKeyValue; +#ifdef REDIS_TEST +int dictTest(int argc, char *argv[], int accurate); +#endif + #ifdef __cplusplus } #endif diff --git a/src/endianconv.c b/src/endianconv.c index 918844e25..98ed405a5 100644 --- a/src/endianconv.c +++ b/src/endianconv.c @@ -105,11 +105,12 @@ uint64_t intrev64(uint64_t v) { #include #define UNUSED(x) (void)(x) -int endianconvTest(int argc, char *argv[]) { +int endianconvTest(int argc, char *argv[], int accurate) { char buf[32]; UNUSED(argc); UNUSED(argv); + UNUSED(accurate); sprintf(buf,"ciaoroma"); memrev16(buf); diff --git a/src/endianconv.h b/src/endianconv.h index 3c8aef14f..08a85d2a9 100644 --- a/src/endianconv.h +++ b/src/endianconv.h @@ -76,7 +76,7 @@ uint64_t intrev64(uint64_t v); #endif #ifdef REDIS_TEST -int endianconvTest(int argc, char *argv[]); +int endianconvTest(int argc, char *argv[], int accurate); #endif #ifdef __cplusplus diff --git a/src/fastlock.cpp b/src/fastlock.cpp index 6e6b6e4d2..789c07caa 100644 --- a/src/fastlock.cpp +++ b/src/fastlock.cpp @@ -70,7 +70,9 @@ #ifdef HAVE_BACKTRACE #include -__attribute__((weak)) void logStackTrace(ucontext_t *) {} +__attribute__((weak)) void logStackTrace(void *, int) { + printf("\tFailed to generate stack trace\n"); +} #endif extern int g_fInCrash; @@ -188,9 +190,7 @@ void printTrace() { #ifdef HAVE_BACKTRACE serverLog(3 /*LL_WARNING*/, "printing backtrace for thread %d", gettid()); - ucontext_t ctxt; - getcontext(&ctxt); - logStackTrace(&ctxt); + logStackTrace(nullptr, 1); #endif } diff --git a/src/help.h b/src/help.h index e91e9b792..c6d7affd6 100644 --- a/src/help.h +++ b/src/help.h @@ -185,7 +185,7 @@ struct commandHelp { 8, "6.2.0" }, { "CLIENT KILL", - "[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [USER username] [ADDR ip:port] [SKIPME yes/no]", + "[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [USER username] [ADDR ip:port] [LADDR ip:port] [SKIPME yes/no]", "Kill the connection of a client", 8, "2.4.0" }, diff --git a/src/intset.c b/src/intset.c index cbb8c49ea..93963209e 100644 --- a/src/intset.c +++ b/src/intset.c @@ -284,7 +284,7 @@ size_t intsetBlobLen(intset *is) { return sizeof(intset)+intrev32ifbe(is->length)*intrev32ifbe(is->encoding); } -/* Validate the integrity of the data stracture. +/* Validate the integrity of the data structure. * when `deep` is 0, only the integrity of the header is validated. * when `deep` is 1, we make sure there are no duplicate or out of order records. */ int intsetValidateIntegrity(const unsigned char *p, size_t size, int deep) { @@ -392,7 +392,7 @@ static void checkConsistency(intset *is) { } #define UNUSED(x) (void)(x) -int intsetTest(int argc, char **argv) { +int intsetTest(int argc, char **argv, int accurate) { uint8_t success; int i; intset *is; @@ -400,6 +400,7 @@ int intsetTest(int argc, char **argv) { UNUSED(argc); UNUSED(argv); + UNUSED(accurate); printf("Value encodings: "); { assert(_intsetValueEncoding(-32768) == INTSET_ENC_INT16); @@ -424,6 +425,7 @@ int intsetTest(int argc, char **argv) { is = intsetAdd(is,4,&success); assert(success); is = intsetAdd(is,4,&success); assert(!success); ok(); + zfree(is); } printf("Large number of random adds: "); { @@ -436,6 +438,7 @@ int intsetTest(int argc, char **argv) { assert(intrev32ifbe(is->length) == inserts); checkConsistency(is); ok(); + zfree(is); } printf("Upgrade from int16 to int32: "); { @@ -447,6 +450,7 @@ int intsetTest(int argc, char **argv) { assert(intsetFind(is,32)); assert(intsetFind(is,65535)); checkConsistency(is); + zfree(is); is = intsetNew(); is = intsetAdd(is,32,NULL); @@ -457,6 +461,7 @@ int intsetTest(int argc, char **argv) { assert(intsetFind(is,-65535)); checkConsistency(is); ok(); + zfree(is); } printf("Upgrade from int16 to int64: "); { @@ -468,6 +473,7 @@ int intsetTest(int argc, char **argv) { assert(intsetFind(is,32)); assert(intsetFind(is,4294967295)); checkConsistency(is); + zfree(is); is = intsetNew(); is = intsetAdd(is,32,NULL); @@ -478,6 +484,7 @@ int intsetTest(int argc, char **argv) { assert(intsetFind(is,-4294967295)); checkConsistency(is); ok(); + zfree(is); } printf("Upgrade from int32 to int64: "); { @@ -489,6 +496,7 @@ int intsetTest(int argc, char **argv) { assert(intsetFind(is,65535)); assert(intsetFind(is,4294967295)); checkConsistency(is); + zfree(is); is = intsetNew(); is = intsetAdd(is,65535,NULL); @@ -499,6 +507,7 @@ int intsetTest(int argc, char **argv) { assert(intsetFind(is,-4294967295)); checkConsistency(is); ok(); + zfree(is); } printf("Stress lookups: "); { @@ -512,6 +521,7 @@ int intsetTest(int argc, char **argv) { for (i = 0; i < num; i++) intsetSearch(is,rand() % ((1<latency_events) == 0 && g_pserver->latency_monitor_threshold == 0) { - report = sdscat(report,"I'm sorry, Dave, I can't do that. Latency monitoring is disabled in this KeyDB instance. You may use \"CONFIG SET latency-monitor-threshold .\" in order to enable it. If we weren't in a deep space mission I'd suggest to take a look at http://redis.io/topics/latency-monitor.\n"); + report = sdscat(report,"I'm sorry, Dave, I can't do that. Latency monitoring is disabled in this KeyDB instance. You may use \"CONFIG SET latency-monitor-threshold .\" in order to enable it. If we weren't in a deep space mission I'd suggest to take a look at https://redis.io/topics/latency-monitor.\n"); return report; } @@ -426,7 +426,7 @@ sds createLatencyReport(void) { } if (advise_slowlog_inspect) { - report = sdscat(report,"- Check your Slow Log to understand what are the commands you are running which are too slow to execute. Please check http://redis.io/commands/slowlog for more information.\n"); + report = sdscat(report,"- Check your Slow Log to understand what are the commands you are running which are too slow to execute. Please check https://redis.io/commands/slowlog for more information.\n"); } /* Intrinsic latency. */ diff --git a/src/listpack.c b/src/listpack.c index a2255f0d7..ee256bad3 100644 --- a/src/listpack.c +++ b/src/listpack.c @@ -908,7 +908,7 @@ int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes) { #undef OUT_OF_RANGE } -/* Validate the integrity of the data stracture. +/* Validate the integrity of the data structure. * when `deep` is 0, only the integrity of the header is validated. * when `deep` is 1, we scan all the entries one by one. */ int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){ diff --git a/src/module.cpp b/src/module.cpp index 1ebf0a5f5..902a1dd84 100644 --- a/src/module.cpp +++ b/src/module.cpp @@ -27,6 +27,30 @@ * POSSIBILITY OF SUCH DAMAGE. */ +/* -------------------------------------------------------------------------- + * Modules API documentation information + * + * The comments in this file are used to generate the API documentation on the + * Redis website. + * + * Each function starting with RM_ and preceded by a block comment is included + * in the API documentation. To hide an RM_ function, put a blank line between + * the comment and the function definition or put the comment inside the + * function body. + * + * The functions are divided into sections. Each section is preceded by a + * documentation block, which is comment block starting with a markdown level 2 + * heading, i.e. a line starting with ##, on the first line of the comment block + * (with the exception of a ----- line which can appear first). Other comment + * blocks, which are not intended for the modules API user, such as this comment + * block, do NOT start with a markdown level 2 heading, so they are included in + * the generated a API documentation. + * + * The documentation comments may contain markdown formatting. Some automatic + * replacements are done, such as the replacement of RM with RedisModule in + * function names. For details, see the script src/modules/gendoc.rb. + * -------------------------------------------------------------------------- */ + #include "server.h" #include "cluster.h" #include "slowlog.h" @@ -177,6 +201,7 @@ typedef struct RedisModuleCtx RedisModuleCtx; #define REDISMODULE_CTX_THREAD_SAFE (1<<4) #define REDISMODULE_CTX_BLOCKED_DISCONNECTED (1<<5) #define REDISMODULE_CTX_MODULE_COMMAND_CALL (1<<6) +#define REDISMODULE_CTX_MULTI_EMITTED (1<<7) /* This represents a Redis key opened with RM_OpenKey(). */ struct RedisModuleKey { @@ -411,7 +436,10 @@ void RM_FreeDict(RedisModuleCtx *ctx, RedisModuleDict *d); void RM_FreeServerInfo(RedisModuleCtx *ctx, RedisModuleServerInfoData *data); /* -------------------------------------------------------------------------- - * Heap allocation raw functions + * ## Heap allocation raw functions + * + * Memory allocated with these functions are taken into account by Redis key + * eviction algorithms and are reported in Redis memory usage information. * -------------------------------------------------------------------------- */ /* Use like malloc(). Memory allocated with this function is reported in @@ -594,13 +622,13 @@ int moduleDelKeyIfEmpty(RedisModuleKey *key) { * defined in the main executable having the same names. * -------------------------------------------------------------------------- */ -/* Lookup the requested module API and store the function pointer into the - * target pointer. The function returns REDISMODULE_ERR if there is no such - * named API, otherwise REDISMODULE_OK. - * - * This function is not meant to be used by modules developer, it is only - * used implicitly by including redismodule.h. */ int RM_GetApi(const char *funcname, void **targetPtrPtr) { + /* Lookup the requested module API and store the function pointer into the + * target pointer. The function returns REDISMODULE_ERR if there is no such + * named API, otherwise REDISMODULE_OK. + * + * This function is not meant to be used by modules developer, it is only + * used implicitly by including redismodule.h. */ dictEntry *he = dictFind(g_pserver->moduleapi, funcname); if (!he) return REDISMODULE_ERR; *targetPtrPtr = dictGetVal(he); @@ -615,17 +643,21 @@ void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) { /* We don't need to do anything here if the context was never used * in order to propagate commands. */ + if (!(ctx->flags & REDISMODULE_CTX_MULTI_EMITTED)) return; + + /* We don't need to do anything here if the server isn't inside + * a transaction. */ if (!g_pserver->propagate_in_transaction) return; - /* If this command is executed from with Lua or MULTI/EXEC we do noy + /* If this command is executed from with Lua or MULTI/EXEC we do not * need to propagate EXEC */ if (serverTL->in_eval || serverTL->in_exec) return; /* Handle the replication of the final EXEC, since whatever a command * emits is always wrapped around MULTI/EXEC. */ - beforePropagateMultiOrExec(0); alsoPropagate(cserver.execCommand,c->db->id,&shared.exec,1, PROPAGATE_AOF|PROPAGATE_REPL); + afterPropagateExec(); /* If this is not a module command context (but is instead a simple * callback context), we have to handle directly the "also propagate" @@ -724,6 +756,14 @@ int moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc, return result->numkeys; } +/* -------------------------------------------------------------------------- + * ## Commands API + * + * These functions are used to implement custom Redis commands. + * + * For examples, see https://redis.io/topics/modules-intro. + * -------------------------------------------------------------------------- */ + /* Return non-zero if a module command, that was declared with the * flag "getkeys-api", is called in a special way to get the keys positions * and not to get executed. Otherwise zero is returned. */ @@ -898,11 +938,15 @@ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc c return REDISMODULE_OK; } -/* Called by RM_Init() to setup the `ctx->module` structure. - * - * This is an internal function, Redis modules developers don't need - * to use it. */ +/* -------------------------------------------------------------------------- + * ## Module information and time measurement + * -------------------------------------------------------------------------- */ + void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int apiver) { + /* Called by RM_Init() to setup the `ctx->module` structure. + * + * This is an internal function, Redis modules developers don't need + * to use it. */ RedisModule *module; if (ctx->module != NULL) return; @@ -968,20 +1012,29 @@ int RM_BlockedClientMeasureTimeEnd(RedisModuleBlockedClient *bc) { * repl-diskless-load to work if enabled. * The module should use RedisModule_IsIOError after reads, before using the * data that was read, and in case of error, propagate it upwards, and also be - * able to release the partially populated value and all it's allocations. */ + * able to release the partially populated value and all it's allocations. + * + * REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED: + * See RM_SignalModifiedKey(). + */ void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) { ctx->module->options = options; } /* Signals that the key is modified from user's perspective (i.e. invalidate WATCH - * and client side caching). */ + * and client side caching). + * + * This is done automatically when a key opened for writing is closed, unless + * the option REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED has been set using + * RM_SetModuleOptions(). +*/ int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) { signalModifiedKey(ctx->client,ctx->client->db,keyname); return REDISMODULE_OK; } /* -------------------------------------------------------------------------- - * Automatic memory management for modules + * ## Automatic memory management for modules * -------------------------------------------------------------------------- */ /* Enable automatic memory management. @@ -1077,7 +1130,7 @@ void autoMemoryCollect(RedisModuleCtx *ctx) { } /* -------------------------------------------------------------------------- - * String objects APIs + * ## String objects APIs * -------------------------------------------------------------------------- */ /* Create a new module string object. The returned string must be freed @@ -1346,14 +1399,6 @@ int RM_StringToLongDouble(const RedisModuleString *str, long double *ld) { * Returns REDISMODULE_OK on success and returns REDISMODULE_ERR if the string * is not a valid string representation of a stream ID. The special IDs "+" and * "-" are allowed. - * - * RedisModuleStreamID is a struct with two 64-bit fields, which is used in - * stream functions and defined as - * - * typedef struct RedisModuleStreamID { - * uint64_t ms; - * uint64_t seq; - * } RedisModuleStreamID; */ int RM_StringToStreamID(const RedisModuleString *str, RedisModuleStreamID *id) { streamID streamid; @@ -1408,13 +1453,15 @@ int RM_StringAppendBuffer(RedisModuleCtx *ctx, RedisModuleString *str, const cha } /* -------------------------------------------------------------------------- - * Reply APIs + * ## Reply APIs + * + * These functions are used for sending replies to the client. * * Most functions always return REDISMODULE_OK so you can use it with * 'return' in order to return from the command implementation with: * * if (... some condition ...) - * return RM_ReplyWithLongLong(ctx,mycount); + * return RedisModule_ReplyWithLongLong(ctx,mycount); * -------------------------------------------------------------------------- */ /* Send an error about the number of arguments given to the command, @@ -1746,7 +1793,7 @@ int RM_ReplyWithLongDouble(RedisModuleCtx *ctx, long double ld) { } /* -------------------------------------------------------------------------- - * Commands replication API + * ## Commands replication API * -------------------------------------------------------------------------- */ /* Helper function to replicate MULTI the first time we replicate something @@ -1755,7 +1802,7 @@ int RM_ReplyWithLongDouble(RedisModuleCtx *ctx, long double ld) { void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { /* Skip this if client explicitly wrap the command with MULTI, or if * the module command was called by a script. */ - if (g_pserver->lua_caller || serverTL->in_exec) return; + if (serverTL->in_eval || serverTL->in_exec) return; /* If we already emitted MULTI return ASAP. */ if (g_pserver->propagate_in_transaction) return; /* If this is a thread safe context, we do not want to wrap commands @@ -1766,10 +1813,12 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { * context, we have to setup the op array for the "also propagate" API * so that RM_Replicate() will work. */ if (!(ctx->flags & REDISMODULE_CTX_MODULE_COMMAND_CALL)) { + serverAssert(ctx->saved_oparray.ops == NULL); ctx->saved_oparray = g_pserver->also_propagate; redisOpArrayInit(&g_pserver->also_propagate); } execCommandPropagateMulti(ctx->client->db->id); + ctx->flags |= REDISMODULE_CTX_MULTI_EMITTED; } /* Replicate the specified command and arguments to slaves and AOF, as effect @@ -1793,7 +1842,7 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { * the AOF or the replicas from the propagation of the specified command. * Otherwise, by default, the command will be propagated in both channels. * - * ## Note about calling this function from a thread safe context: + * #### Note about calling this function from a thread safe context: * * Normally when you call this function from the callback implementing a * module command, or any other callback provided by the Redis Module API, @@ -1805,7 +1854,7 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { * and the command specified is inserted in the AOF and replication stream * immediately. * - * ## Return value + * #### Return value * * The command returns REDISMODULE_ERR if the format specifiers are invalid * or the command name does not belong to a known command. */ @@ -1869,7 +1918,7 @@ int RM_ReplicateVerbatim(RedisModuleCtx *ctx) { } /* -------------------------------------------------------------------------- - * DB and Key APIs -- Generic API + * ## DB and Key APIs -- Generic API * -------------------------------------------------------------------------- */ /* Return the ID of the current client calling the currently active module @@ -2145,7 +2194,7 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { flags |= REDISMODULE_CTX_FLAGS_LOADING; /* Maxmemory and eviction policy */ - if (g_pserver->maxmemory > 0) { + if (g_pserver->maxmemory > 0 && (!listLength(g_pserver->masters) || !g_pserver->repl_slave_ignore_maxmemory)) { flags |= REDISMODULE_CTX_FLAGS_MAXMEMORY; if (g_pserver->maxmemory_policy != MAXMEMORY_NO_EVICTION) @@ -2400,7 +2449,7 @@ mstime_t RM_GetExpire(RedisModuleKey *key) { * The function returns REDISMODULE_OK on success or REDISMODULE_ERR if * the key was not open for writing or is an empty key. */ int RM_SetExpire(RedisModuleKey *key, mstime_t expire) { - if (!(key->mode & REDISMODULE_WRITE) || key->value == NULL) + if (!(key->mode & REDISMODULE_WRITE) || key->value == NULL || (expire < 0 && expire != REDISMODULE_NO_EXPIRE)) return REDISMODULE_ERR; if (expire != REDISMODULE_NO_EXPIRE) { expire += mstime(); @@ -2411,6 +2460,36 @@ int RM_SetExpire(RedisModuleKey *key, mstime_t expire) { return REDISMODULE_OK; } +/* Return the key expire value, as absolute Unix timestamp. + * If no TTL is associated with the key or if the key is empty, + * REDISMODULE_NO_EXPIRE is returned. */ +mstime_t RM_GetAbsExpire(RedisModuleKey *key) { + auto expire = getExpire(key->db,key->key); + if (expire == nullptr || key->value == NULL) + return REDISMODULE_NO_EXPIRE; + return expire->when(); +} + +/* Set a new expire for the key. If the special expire + * REDISMODULE_NO_EXPIRE is set, the expire is cancelled if there was + * one (the same as the PERSIST command). + * + * Note that the expire must be provided as a positive integer representing + * the absolute Unix timestamp the key should have. + * + * The function returns REDISMODULE_OK on success or REDISMODULE_ERR if + * the key was not open for writing or is an empty key. */ +int RM_SetAbsExpire(RedisModuleKey *key, mstime_t expire) { + if (!(key->mode & REDISMODULE_WRITE) || key->value == NULL || (expire < 0 && expire != REDISMODULE_NO_EXPIRE)) + return REDISMODULE_ERR; + if (expire != REDISMODULE_NO_EXPIRE) { + setExpire(key->ctx->client,key->db,key->key,nullptr/*subkey*/,expire); + } else { + removeExpire(key->db,key->key); + } + return REDISMODULE_OK; +} + /* Performs similar operation to FLUSHALL, and optionally start a new AOF file (if enabled) * If restart_aof is true, you must make sure the command that triggered this call is not * propagated to the AOF file. @@ -2434,7 +2513,9 @@ RedisModuleString *RM_RandomKey(RedisModuleCtx *ctx) { } /* -------------------------------------------------------------------------- - * Key API for String type + * ## Key API for String type + * + * See also RM_ValueLength(), which returns the length of a string. * -------------------------------------------------------------------------- */ /* If the key is open for writing, set the specified string 'str' as the @@ -2544,7 +2625,9 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) { } /* -------------------------------------------------------------------------- - * Key API for List type + * ## Key API for List type + * + * See also RM_ValueLength(), which returns the length of a list. * -------------------------------------------------------------------------- */ /* Push an element into a list, on head or tail depending on 'where' argument. @@ -2582,26 +2665,28 @@ RedisModuleString *RM_ListPop(RedisModuleKey *key, int where) { } /* -------------------------------------------------------------------------- - * Key API for Sorted Set type + * ## Key API for Sorted Set type + * + * See also RM_ValueLength(), which returns the length of a sorted set. * -------------------------------------------------------------------------- */ /* Conversion from/to public flags of the Modules API and our private flags, * so that we have everything decoupled. */ int moduleZsetAddFlagsToCoreFlags(int flags) { int retflags = 0; - if (flags & REDISMODULE_ZADD_XX) retflags |= ZADD_XX; - if (flags & REDISMODULE_ZADD_NX) retflags |= ZADD_NX; - if (flags & REDISMODULE_ZADD_GT) retflags |= ZADD_GT; - if (flags & REDISMODULE_ZADD_LT) retflags |= ZADD_LT; + if (flags & REDISMODULE_ZADD_XX) retflags |= ZADD_IN_XX; + if (flags & REDISMODULE_ZADD_NX) retflags |= ZADD_IN_NX; + if (flags & REDISMODULE_ZADD_GT) retflags |= ZADD_IN_GT; + if (flags & REDISMODULE_ZADD_LT) retflags |= ZADD_IN_LT; return retflags; } /* See previous function comment. */ int moduleZsetAddFlagsFromCoreFlags(int flags) { int retflags = 0; - if (flags & ZADD_ADDED) retflags |= REDISMODULE_ZADD_ADDED; - if (flags & ZADD_UPDATED) retflags |= REDISMODULE_ZADD_UPDATED; - if (flags & ZADD_NOP) retflags |= REDISMODULE_ZADD_NOP; + if (flags & ZADD_OUT_ADDED) retflags |= REDISMODULE_ZADD_ADDED; + if (flags & ZADD_OUT_UPDATED) retflags |= REDISMODULE_ZADD_UPDATED; + if (flags & ZADD_OUT_NOP) retflags |= REDISMODULE_ZADD_NOP; return retflags; } @@ -2638,16 +2723,16 @@ int moduleZsetAddFlagsFromCoreFlags(int flags) { * * 'score' double value is not a number (NaN). */ int RM_ZsetAdd(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr) { - int flags = 0; + int in_flags = 0, out_flags = 0; if (!(key->mode & REDISMODULE_WRITE)) return REDISMODULE_ERR; if (key->value && key->value->type != OBJ_ZSET) return REDISMODULE_ERR; if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_ZSET); - if (flagsptr) flags = moduleZsetAddFlagsToCoreFlags(*flagsptr); - if (zsetAdd(key->value,score,szFromObj(ele),&flags,NULL) == 0) { + if (flagsptr) in_flags = moduleZsetAddFlagsToCoreFlags(*flagsptr); + if (zsetAdd(key->value,score,szFromObj(ele),in_flags,&out_flags,NULL) == 0) { if (flagsptr) *flagsptr = 0; return REDISMODULE_ERR; } - if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(flags); + if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(out_flags); return REDISMODULE_OK; } @@ -2665,22 +2750,17 @@ int RM_ZsetAdd(RedisModuleKey *key, double score, RedisModuleString *ele, int *f * with the new score of the element after the increment, if no error * is returned. */ int RM_ZsetIncrby(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr, double *newscore) { - int flags = 0; + int in_flags = 0, out_flags = 0; if (!(key->mode & REDISMODULE_WRITE)) return REDISMODULE_ERR; if (key->value && key->value->type != OBJ_ZSET) return REDISMODULE_ERR; if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_ZSET); - if (flagsptr) flags = moduleZsetAddFlagsToCoreFlags(*flagsptr); - flags |= ZADD_INCR; - if (zsetAdd(key->value,score,szFromObj(ele),&flags,newscore) == 0) { + if (flagsptr) in_flags = moduleZsetAddFlagsToCoreFlags(*flagsptr); + in_flags |= ZADD_IN_INCR; + if (zsetAdd(key->value,score,szFromObj(ele),in_flags,&out_flags,newscore) == 0) { if (flagsptr) *flagsptr = 0; return REDISMODULE_ERR; } - /* zsetAdd() may signal back that the resulting score is not a number. */ - if (flagsptr && (*flagsptr & ZADD_NAN)) { - *flagsptr = 0; - return REDISMODULE_ERR; - } - if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(flags); + if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(out_flags); return REDISMODULE_OK; } @@ -2730,7 +2810,7 @@ int RM_ZsetScore(RedisModuleKey *key, RedisModuleString *ele, double *score) { } /* -------------------------------------------------------------------------- - * Key API for Sorted Set iterator + * ## Key API for Sorted Set iterator * -------------------------------------------------------------------------- */ void zsetKeyReset(RedisModuleKey *key) { @@ -3037,7 +3117,9 @@ int RM_ZsetRangePrev(RedisModuleKey *key) { } /* -------------------------------------------------------------------------- - * Key API for Hash type + * ## Key API for Hash type + * + * See also RM_ValueLength(), which returns the number of fields in a hash. * -------------------------------------------------------------------------- */ /* Set the field of the specified hash field to the specified value. @@ -3272,7 +3354,20 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) { } /* -------------------------------------------------------------------------- - * Key API for the stream type. + * ## Key API for Stream type + * + * For an introduction to streams, see https://redis.io/topics/streams-intro. + * + * The type RedisModuleStreamID, which is used in stream functions, is a struct + * with two 64-bit fields and is defined as + * + * typedef struct RedisModuleStreamID { + * uint64_t ms; + * uint64_t seq; + * } RedisModuleStreamID; + * + * See also RM_ValueLength(), which returns the length of a stream, and the + * conversion functions RM_StringToStreamID() and RM_CreateStringFromStreamID(). * -------------------------------------------------------------------------- */ /* Adds an entry to a stream. Like XADD without trimming. @@ -3439,8 +3534,8 @@ int RM_StreamDelete(RedisModuleKey *key, RedisModuleStreamID *id) { * // * // ... Do stuff ... * // - * RedisModule_Free(field); - * RedisModule_Free(value); + * RedisModule_FreeString(ctx, field); + * RedisModule_FreeString(ctx, value); * } * } * RedisModule_StreamIteratorStop(key); @@ -3721,7 +3816,9 @@ long long RM_StreamTrimByID(RedisModuleKey *key, int flags, RedisModuleStreamID } /* -------------------------------------------------------------------------- - * Redis <-> Modules generic Call() API + * ## Calling Redis commands from modules + * + * RM_Call() sends a command to Redis. The remaining functions handle the reply. * -------------------------------------------------------------------------- */ /* Create a new RedisModuleCallReply object. The processing of the reply @@ -4073,6 +4170,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch int replicate = 0; /* Replicate this command? */ int call_flags; sds proto = nullptr; + int prev_replication_allowed; /* Handle arguments. */ va_start(ap, fmt); @@ -4150,20 +4248,30 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch } } - /* If we are using single commands replication, we need to wrap what - * we propagate into a MULTI/EXEC block, so that it will be atomic like - * a Lua script in the context of AOF and slaves. */ - if (replicate) moduleReplicateMultiIfNeeded(ctx); + /* We need to use a global replication_allowed flag in order to prevent + * replication of nested RM_Calls. Example: + * 1. module1.foo does RM_Call of module2.bar without replication (i.e. no '!') + * 2. module2.bar internally calls RM_Call of INCR with '!' + * 3. at the end of module1.foo we call RM_ReplicateVerbatim + * We want the replica/AOF to see only module1.foo and not the INCR from module2.bar */ + prev_replication_allowed = g_pserver->replication_allowed; + g_pserver->replication_allowed = replicate && g_pserver->replication_allowed; /* Run the command */ call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS | CMD_CALL_NOWRAP; if (replicate) { + /* If we are using single commands replication, we need to wrap what + * we propagate into a MULTI/EXEC block, so that it will be atomic like + * a Lua script in the context of AOF and slaves. */ + moduleReplicateMultiIfNeeded(ctx); + if (!(flags & REDISMODULE_ARGV_NO_AOF)) call_flags |= CMD_CALL_PROPAGATE_AOF; if (!(flags & REDISMODULE_ARGV_NO_REPLICAS)) call_flags |= CMD_CALL_PROPAGATE_REPL; } call(c,call_flags); + g_pserver->replication_allowed = prev_replication_allowed; serverAssert((c->flags & CLIENT_BLOCKED) == 0); @@ -4204,7 +4312,7 @@ const char *RM_CallReplyProto(RedisModuleCallReply *reply, size_t *len) { } /* -------------------------------------------------------------------------- - * Modules data types + * ## Modules data types * * When String DMA or using existing data structures is not enough, it is * possible to create new data types from scratch and export them to @@ -4347,6 +4455,12 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) { } } +/* Return the name of the module that owns the specified moduleType. */ +const char *moduleTypeModuleName(moduleType *mt) { + if (!mt || !mt->module) return NULL; + return mt->module->name; +} + /* Create a copy of a module type value using the copy callback. If failed * or not supported, produce an error reply and return NULL. */ @@ -4562,7 +4676,7 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { } /* -------------------------------------------------------------------------- - * RDB loading and saving functions + * ## RDB loading and saving functions * -------------------------------------------------------------------------- */ /* Called when there is a load error in the context of a module. On some @@ -4878,7 +4992,7 @@ ssize_t rdbSaveModulesAux(rio *rdb, int when) { } /* -------------------------------------------------------------------------- - * Key digest API (DEBUG DIGEST interface for modules types) + * ## Key digest API (DEBUG DIGEST interface for modules types) * -------------------------------------------------------------------------- */ /* Add a new element to the digest. This function can be called multiple times @@ -4999,7 +5113,7 @@ RedisModuleString *RM_SaveDataTypeToString(RedisModuleCtx *ctx, void *data, cons } /* -------------------------------------------------------------------------- - * AOF API for modules data types + * ## AOF API for modules data types * -------------------------------------------------------------------------- */ /* Emits a command into the AOF during the AOF rewriting process. This function @@ -5054,7 +5168,7 @@ void RM_EmitAOF(RedisModuleIO *io, const char *cmdname, const char *fmt, ...) { } /* -------------------------------------------------------------------------- - * IO context handling + * ## IO context handling * -------------------------------------------------------------------------- */ RedisModuleCtx *RM_GetContextFromIO(RedisModuleIO *io) { @@ -5081,7 +5195,7 @@ const RedisModuleString *RM_GetKeyNameFromModuleKey(RedisModuleKey *key) { } /* -------------------------------------------------------------------------- - * Logging + * ## Logging * -------------------------------------------------------------------------- */ /* This is the low level function implementing both: @@ -5112,10 +5226,10 @@ void moduleLogRaw(RedisModule *module, const char *levelstr, const char *fmt, va * printf-alike specifiers, while level is a string describing the log * level to use when emitting the log, and must be one of the following: * - * * "debug" - * * "verbose" - * * "notice" - * * "warning" + * * "debug" (`REDISMODULE_LOGLEVEL_DEBUG`) + * * "verbose" (`REDISMODULE_LOGLEVEL_VERBOSE`) + * * "notice" (`REDISMODULE_LOGLEVEL_NOTICE`) + * * "warning" (`REDISMODULE_LOGLEVEL_WARNING`) * * If the specified log level is invalid, verbose is used by default. * There is a fixed limit to the length of the log line this function is able @@ -5166,7 +5280,10 @@ void RM_LatencyAddSample(const char *event, mstime_t latency) { } /* -------------------------------------------------------------------------- - * Blocking clients from modules + * ## Blocking clients from modules + * + * For a guide about blocking commands in modules, see + * https://redis.io/topics/modules-blocking-ops. * -------------------------------------------------------------------------- */ /* Readable handler for the awake pipe. We do nothing here, the awake bytes @@ -5227,11 +5344,6 @@ void unblockClientFromModule(client *c) { moduleUnblockClient(c); bc->client = NULL; - /* Reset the client for a new query since, for blocking commands implemented - * into modules, we do not it immediately after the command returns (and - * the client blocks) in order to be still able to access the argument - * vector from callbacks. */ - resetClient(c); } /* Block a client in the context of a module: this function implements both @@ -5651,6 +5763,12 @@ void moduleHandleBlockedClients(int iel) { * API to unblock the client and the memory will be released. */ void moduleBlockedClientTimedOut(client *c) { RedisModuleBlockedClient *bc = (RedisModuleBlockedClient*)c->bpop.module_blocked_handle; + + /* Protect against re-processing: don't serve clients that are already + * in the unblocking list for any reason (including RM_UnblockClient() + * explicit call). See #6798. */ + if (bc->unblocked) return; + RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.flags |= REDISMODULE_CTX_BLOCKED_TIMEOUT; ctx.module = bc->module; @@ -5707,7 +5825,7 @@ int RM_BlockedClientDisconnected(RedisModuleCtx *ctx) { } /* -------------------------------------------------------------------------- - * Thread Safe Contexts + * ## Thread Safe Contexts * -------------------------------------------------------------------------- */ /* Return a context which can be used inside threads to make Redis context @@ -5907,7 +6025,7 @@ int moduleGILAcquiredByModule(void) { /* -------------------------------------------------------------------------- - * Module Keyspace Notifications API + * ## Module Keyspace Notifications API * -------------------------------------------------------------------------- */ /* Subscribe to keyspace notifications. This is a low-level version of the @@ -5931,6 +6049,7 @@ int moduleGILAcquiredByModule(void) { * - REDISMODULE_NOTIFY_EXPIRED: Expiration events * - REDISMODULE_NOTIFY_EVICTED: Eviction events * - REDISMODULE_NOTIFY_STREAM: Stream events + * - REDISMODULE_NOTIFY_MODULE: Module types events * - REDISMODULE_NOTIFY_KEYMISS: Key-miss events * - REDISMODULE_NOTIFY_ALL: All events (Excluding REDISMODULE_NOTIFY_KEYMISS) * - REDISMODULE_NOTIFY_LOADED: A special notification available only for modules, @@ -6040,7 +6159,7 @@ void moduleUnsubscribeNotifications(RedisModule *module) { } /* -------------------------------------------------------------------------- - * Modules Cluster API + * ## Modules Cluster API * -------------------------------------------------------------------------- */ /* The Cluster message callback function pointer type. */ @@ -6295,7 +6414,7 @@ void RM_SetClusterFlags(RedisModuleCtx *ctx, uint64_t flags) { } /* -------------------------------------------------------------------------- - * Modules Timers API + * ## Modules Timers API * * Module timers are an high precision "green timers" abstraction where * every module can register even millions of timers without problems, even if @@ -6485,7 +6604,7 @@ int RM_GetTimerInfo(RedisModuleCtx *ctx, RedisModuleTimerID id, uint64_t *remain } /* -------------------------------------------------------------------------- - * Modules ACL API + * ## Modules ACL API * * Implements a hook into the authentication and authorization within Redis. * --------------------------------------------------------------------------*/ @@ -6715,7 +6834,7 @@ RedisModuleString *RM_GetClientCertificate(RedisModuleCtx *ctx, uint64_t client_ } /* -------------------------------------------------------------------------- - * Modules Dictionary API + * ## Modules Dictionary API * * Implements a sorted dictionary (actually backed by a radix tree) with * the usual get / set / del / num-items API, together with an iterator @@ -6969,7 +7088,7 @@ int RM_DictCompare(RedisModuleDictIter *di, const char *op, RedisModuleString *k /* -------------------------------------------------------------------------- - * Modules Info fields + * ## Modules Info fields * -------------------------------------------------------------------------- */ int RM_InfoEndDictField(RedisModuleInfoCtx *ctx); @@ -7283,7 +7402,7 @@ double RM_ServerInfoGetFieldDouble(RedisModuleServerInfoData *data, const char* } /* -------------------------------------------------------------------------- - * Modules utility APIs + * ## Modules utility APIs * -------------------------------------------------------------------------- */ /* Return random bytes using SHA1 in counter mode with a /dev/urandom @@ -7302,7 +7421,7 @@ void RM_GetRandomHexChars(char *dst, size_t len) { } /* -------------------------------------------------------------------------- - * Modules API exporting / importing + * ## Modules API exporting / importing * -------------------------------------------------------------------------- */ /* This function is called by a module in order to export some API with a @@ -7439,7 +7558,7 @@ int moduleUnregisterFilters(RedisModule *module) { } /* -------------------------------------------------------------------------- - * Module Command Filter API + * ## Module Command Filter API * -------------------------------------------------------------------------- */ /* Register a new command filter function. @@ -7652,7 +7771,7 @@ float RM_GetUsedMemoryRatio(){ } /* -------------------------------------------------------------------------- - * Scanning keyspace and hashes + * ## Scanning keyspace and hashes * -------------------------------------------------------------------------- */ typedef void (*RedisModuleScanCB)(RedisModuleCtx *ctx, RedisModuleString *keyname, RedisModuleKey *key, void *privdata); @@ -7923,7 +8042,7 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc /* -------------------------------------------------------------------------- - * Module fork API + * ## Module fork API * -------------------------------------------------------------------------- */ /* Create a background child process with the current frozen snaphost of the @@ -7984,7 +8103,7 @@ int TerminateModuleForkChild(int child_pid, int wait) { serverLog(LL_VERBOSE,"Killing running module fork child: %ld", (long) g_pserver->child_pid); if (kill(g_pserver->child_pid,SIGUSR1) != -1 && wait) { - while(wait4(g_pserver->child_pid,&statloc,0,NULL) != + while(waitpid(g_pserver->child_pid,&statloc,0) != g_pserver->child_pid); } /* Reset the buffer accumulating changes while the child saves. */ @@ -8017,7 +8136,7 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { } /* -------------------------------------------------------------------------- - * Server hooks implementation + * ## Server hooks implementation * -------------------------------------------------------------------------- */ /* Register to be notified, via a callback, when the specified server event @@ -8899,6 +9018,10 @@ size_t moduleCount(void) { return dictSize(modules); } +/* -------------------------------------------------------------------------- + * ## Key eviction API + * -------------------------------------------------------------------------- */ + /* Set the key last access time for LRU based eviction. not relevant if the * servers's maxmemory policy is LFU based. Value is idle time in milliseconds. * returns REDISMODULE_OK if the LRU was updated, REDISMODULE_ERR otherwise. */ @@ -8951,6 +9074,10 @@ int RM_GetLFU(RedisModuleKey *key, long long *lfu_freq) { return REDISMODULE_OK; } +/* -------------------------------------------------------------------------- + * ## Miscellaneous APIs + * -------------------------------------------------------------------------- */ + /** * Returns the full ContextFlags mask, using the return value * the module can check if a certain set of flags are supported @@ -9099,6 +9226,10 @@ int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, return res; } +/* -------------------------------------------------------------------------- + * ## Defrag API + * -------------------------------------------------------------------------- */ + /* The defrag context, used to manage state during calls to the data type * defrag callback. */ diff --git a/src/modules/gendoc.rb b/src/modules/gendoc.rb index 2fd2ec5d7..f83b1ad9d 100644 --- a/src/modules/gendoc.rb +++ b/src/modules/gendoc.rb @@ -1,3 +1,4 @@ +# coding: utf-8 # gendoc.rb -- Converts the top-comments inside module.c to modules API # reference documentation in markdown format. @@ -21,15 +22,48 @@ def markdown(s) l = l.gsub(/(?\n\n" + puts "### `#{name}`\n\n" puts " #{proto}\n" comment = "" while true @@ -50,13 +88,87 @@ def docufy(src,i) puts comment+"\n\n" end +# Print a comment from line until */ is found, as markdown. +def section_doc(src, i) + name = get_section_heading(src, i) + comment = "\n\n" + while true + # append line, except if it's a horizontal divider + comment = comment + src[i] if src[i] !~ /^[\/ ]?\*{1,2} ?-{50,}/ + break if src[i] =~ /\*\// + i = i+1 + end + comment = markdown(comment) + puts comment+"\n\n" +end + +# generates an id suitable for links within the page +def section_name_to_id(name) + return "section-" + + name.strip.downcase.gsub(/[^a-z0-9]+/, '-').gsub(/^-+|-+$/, '') +end + +# Returns the name of the first section heading in the comment block for which +# is_section_doc(src, i) is true +def get_section_heading(src, i) + if src[i] =~ /^\/\*\*? \#+ *(.*)/ + heading = $1 + elsif src[i+1] =~ /^ ?\* \#+ *(.*)/ + heading = $1 + end + return heading.gsub(' -- ', ' – ') +end + +# Returns true if the line is the start of a generic documentation section. Such +# section must start with the # symbol, i.e. a markdown heading, on the first or +# the second line. +def is_section_doc(src, i) + return src[i] =~ /^\/\*\*? \#/ || + (src[i] =~ /^\/\*/ && src[i+1] =~ /^ ?\* \#/) +end + +def is_func_line(src, i) + line = src[i] + return line =~ /RM_/ && + line[0] != ' ' && line[0] != '#' && line[0] != '/' && + src[i-1] =~ /\*\// +end + puts "# Modules API reference\n\n" puts "\n\n" -src = File.open("../module.c").to_a -src.each_with_index{|line,i| - if line =~ /RM_/ && line[0] != ' ' && line[0] != '#' && line[0] != '/' - if src[i-1] =~ /\*\// - docufy(src,i) - end +src = File.open(File.dirname(__FILE__) ++ "/../module.c").to_a + +# Build function index +$index = {} +src.each_with_index do |line,i| + if is_func_line(src, i) + line =~ /RM_([A-z0-9]+)/ + name = "RedisModule_#{$1}" + $index[name] = true end -} +end + +# Print TOC +puts "## Sections\n\n" +src.each_with_index do |_line,i| + if is_section_doc(src, i) + name = get_section_heading(src, i) + puts "* [#{name}](\##{section_name_to_id(name)})\n" + end +end +puts "* [Function index](#section-function-index)\n\n" + +# Docufy: Print function prototype and markdown docs +src.each_with_index do |_line,i| + if is_func_line(src, i) + docufy(src, i) + elsif is_section_doc(src, i) + section_doc(src, i) + end +end + +# Print function index +puts "\n\n" +puts "## Function index\n\n" +$index.keys.sort.each{|x| puts "* [`#{x}`](\##{x})\n"} +puts "\n" diff --git a/src/multi.cpp b/src/multi.cpp index fa02741cf..8ccec0a67 100644 --- a/src/multi.cpp +++ b/src/multi.cpp @@ -116,34 +116,34 @@ void discardCommand(client *c) { addReply(c,shared.ok); } -void beforePropagateMultiOrExec(int multi) { - if (multi) { - /* Propagating MULTI */ - serverAssert(!g_pserver->propagate_in_transaction); - g_pserver->propagate_in_transaction = 1; - } else { - /* Propagating EXEC */ - serverAssert(g_pserver->propagate_in_transaction == 1); - g_pserver->propagate_in_transaction = 0; - } +void beforePropagateMulti() { + /* Propagating MULTI */ + serverAssert(!g_pserver->propagate_in_transaction); + g_pserver->propagate_in_transaction = 1; +} + +void afterPropagateExec() { + /* Propagating EXEC */ + serverAssert(g_pserver->propagate_in_transaction == 1); + g_pserver->propagate_in_transaction = 0; } /* Send a MULTI command to all the slaves and AOF file. Check the execCommand * implementation for more information. */ void execCommandPropagateMulti(int dbid) { - beforePropagateMultiOrExec(1); + beforePropagateMulti(); propagate(cserver.multiCommand,dbid,&shared.multi,1, PROPAGATE_AOF|PROPAGATE_REPL); } void execCommandPropagateExec(int dbid) { - beforePropagateMultiOrExec(0); propagate(cserver.execCommand,dbid,&shared.exec,1, PROPAGATE_AOF|PROPAGATE_REPL); + afterPropagateExec(); } /* Aborts a transaction, with a specific error message. - * The transaction is always aboarted with -EXECABORT so that the client knows + * The transaction is always aborted with -EXECABORT so that the client knows * the server exited the multi state, but the actual reason for the abort is * included too. * Note: 'error' may or may not end with \r\n. see addReplyErrorFormat. */ @@ -206,11 +206,9 @@ void execCommand(client *c) { c->cmd = c->mstate.commands[j].cmd; /* ACL permissions are also checked at the time of execution in case - * they were changed after the commands were ququed. */ + * they were changed after the commands were queued. */ int acl_errpos; - int acl_retval = ACLCheckCommandPerm(c,&acl_errpos); - if (acl_retval == ACL_OK && c->cmd->proc == publishCommand) - acl_retval = ACLCheckPubsubPerm(c,1,1,0,&acl_errpos); + int acl_retval = ACLCheckAllPerm(c,&acl_errpos); if (acl_retval != ACL_OK) { const char *reason; switch (acl_retval) { @@ -221,7 +219,8 @@ void execCommand(client *c) { reason = "no permission to touch the specified keys"; break; case ACL_DENIED_CHANNEL: - reason = "no permission to publish to the specified channel"; + reason = "no permission to access one of the channels used " + "as arguments"; break; default: reason = "no permission"; @@ -261,7 +260,6 @@ void execCommand(client *c) { if (g_pserver->propagate_in_transaction) { int is_master = listLength(g_pserver->masters) == 0; g_pserver->dirty++; - beforePropagateMultiOrExec(0); /* If inside the MULTI/EXEC block this instance was suddenly * switched from master to replica (using the SLAVEOF command), the * initial MULTI was propagated into the replication backlog, but the @@ -271,6 +269,7 @@ void execCommand(client *c) { const char *execcmd = "*1\r\n$4\r\nEXEC\r\n"; feedReplicationBacklog(execcmd,strlen(execcmd)); } + afterPropagateExec(); } serverTL->in_exec = 0; diff --git a/src/networking.cpp b/src/networking.cpp index 6c0637f6c..c622dcc55 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -169,6 +169,7 @@ client *createClient(connection *conn, int iel) { c->read_reploff = 0; c->repl_ack_off = 0; c->repl_ack_time = 0; + c->repl_last_partial_write = 0; c->slave_listening_port = 0; c->slave_addr = NULL; c->slave_capa = SLAVE_CAPA_NONE; @@ -404,8 +405,9 @@ void _addReplyProtoToList(client *c, const char *s, size_t len) { memcpy(tail->buf(), s, len); listAddNodeTail(c->reply, tail); c->reply_bytes += tail->size; + + asyncCloseClientOnOutputBufferLimitReached(c); } - asyncCloseClientOnOutputBufferLimitReached(c); } /* ----------------------------------------------------------------------------- @@ -684,7 +686,7 @@ void *addReplyDeferredLen(client *c) { void setDeferredReply(client *c, void *node, const char *s, size_t length) { listNode *ln = (listNode*)node; - clientReplyBlock *next; + clientReplyBlock *next, *prev; /* Abort when *node is NULL: when the client should not accept writes * we return NULL in addReplyDeferredLen() */ @@ -693,14 +695,31 @@ void setDeferredReply(client *c, void *node, const char *s, size_t length) { /* Normally we fill this dummy NULL node, added by addReplyDeferredLen(), * with a new buffer structure containing the protocol needed to specify - * the length of the array following. However sometimes when there is - * little memory to move, we may instead remove this NULL node, and prefix - * our protocol in the node immediately after to it, in order to save a - * write(2) syscall later. Conditions needed to do it: + * the length of the array following. However sometimes there might be room + * in the previous/next node so we can instead remove this NULL node, and + * suffix/prefix our data in the node immediately before/after it, in order + * to save a write(2) syscall later. Conditions needed to do it: * + * - The prev node is non-NULL and has space in it or * - The next node is non-NULL, * - It has enough room already allocated * - And not too large (avoid large memmove) */ + if (ln->prev != NULL && (prev = (clientReplyBlock*)listNodeValue(ln->prev)) && + prev->size - prev->used > 0) + { + size_t len_to_copy = prev->size - prev->used; + if (len_to_copy > length) + len_to_copy = length; + memcpy(prev->buf() + prev->used, s, len_to_copy); + prev->used += len_to_copy; + length -= len_to_copy; + if (length == 0) { + listDelNode(c->reply, ln); + return; + } + s += len_to_copy; + } + if (ln->next != NULL && (next = (clientReplyBlock*)listNodeValue(ln->next)) && next->size - next->used >= length && next->used < PROTO_REPLY_CHUNK_BYTES * 4) @@ -718,8 +737,9 @@ void setDeferredReply(client *c, void *node, const char *s, size_t length) { memcpy(buf->buf(), s, length); listNodeValue(ln) = buf; c->reply_bytes += buf->size; + + asyncCloseClientOnOutputBufferLimitReached(c); } - asyncCloseClientOnOutputBufferLimitReached(c); } /* Populate the length object and try gluing it to the next chunk. */ @@ -1795,9 +1815,7 @@ int writeToClient(client *c, int handler_installed) { g_pserver->stat_net_output_bytes += totwritten; if (nwritten == -1) { - if (connGetState(c->conn) == CONN_STATE_CONNECTED) { - nwritten = 0; - } else { + if (connGetState(c->conn) != CONN_STATE_CONNECTED) { serverLog(LL_VERBOSE, "Error writing to client: %s", connGetLastError(c->conn)); freeClientAsync(c); @@ -2022,6 +2040,9 @@ void resetClient(client *c) { c->flags |= CLIENT_REPLY_SKIP; c->flags &= ~CLIENT_REPLY_SKIP_NEXT; } + + /* Always clear the prevent logging field. */ + c->flags &= ~CLIENT_PREVENT_LOGGING; } /* This function is used when we want to re-enter the event loop but there @@ -2333,13 +2354,10 @@ void commandProcessed(client *c, int flags) { c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos; } - /* Don't reset the client structure for clients blocked in a - * module blocking command, so that the reply callback will - * still be able to access the client argv and argc field. - * The client will be reset in unblockClientFromModule(). */ - if (!(c->flags & CLIENT_BLOCKED) || - (c->btype != BLOCKED_MODULE && c->btype != BLOCKED_PAUSE)) - { + /* Don't reset the client structure for blocked clients, so that the reply + * callback will still be able to access the client argv and argc fields. + * The client will be reset in unblockClient(). */ + if (!(c->flags & CLIENT_BLOCKED)) { resetClient(c); } @@ -2373,6 +2391,7 @@ void commandProcessed(client *c, int flags) { * of processing the command, otherwise C_OK is returned. */ int processCommandAndResetClient(client *c, int flags) { int deadclient = 0; + client *old_client = serverTL->current_client; serverTL->current_client = c; serverAssert(GlobalLocksAcquired()); @@ -2380,7 +2399,14 @@ int processCommandAndResetClient(client *c, int flags) { commandProcessed(c, flags); } if (serverTL->current_client == NULL) deadclient = 1; - serverTL->current_client = NULL; + /* + * Restore the old client, this is needed because when a script + * times out, we will get into this code from processEventsWhileBlocked. + * Which will cause to set the server.current_client. If not restored + * we will return 1 to our caller which will falsely indicate the client + * is dead and will stop reading from its buffer. + */ + serverTL->current_client = old_client; /* performEvictions may flush slave output buffers. This may * result in a replica, that may be the active client, to be * freed. */ @@ -3389,6 +3415,7 @@ void helloCommand(client *c) { int moreargs = (c->argc-1) - j; const char *opt = (const char*)ptrFromObj(c->argv[j]); if (!strcasecmp(opt,"AUTH") && moreargs >= 2) { + preventCommandLogging(c); if (ACLAuthenticateUser(c, c->argv[j+1], c->argv[j+2]) == C_ERR) { addReplyError(c,"-WRONGPASS invalid username-password pair or user is disabled."); return; @@ -3457,7 +3484,7 @@ void securityWarningCommand(client *c) { static time_t logged_time; time_t now = time(NULL); - if (labs(now-logged_time) > 60) { + if (llabs(now-logged_time) > 60) { serverLog(LL_WARNING,"Possible SECURITY ATTACK detected. It looks like somebody is sending POST or Host: commands to KeyDB. This is likely due to an attacker attempting to use Cross Protocol Scripting to compromise your KeyDB instance. Connection aborted."); logged_time = now; } @@ -3749,6 +3776,7 @@ void unpauseClients(void) { listRewind(g_pserver->paused_clients,&li); while ((ln = listNext(&li)) != NULL) { c = (client*)listNodeValue(ln); + std::unique_lock ul(c->lock); unblockClient(c); } } @@ -3762,6 +3790,8 @@ int areClientsPaused(void) { * if it has. Also returns true if clients are now paused and false * otherwise. */ int checkClientPauseTimeoutAndReturnIfPaused(void) { + if (!areClientsPaused()) + return 0; if (g_pserver->client_pause_end_time < g_pserver->mstime) { unpauseClients(); } @@ -3813,6 +3843,7 @@ void processEventsWhileBlocked(int iel) { g_pserver->repl_batch_offStart = -1; } + long long eventsCount = 0; aeReleaseLock(); serverAssert(!GlobalLocksAcquired()); try @@ -3825,8 +3856,8 @@ void processEventsWhileBlocked(int iel) { AE_CALL_BEFORE_SLEEP|AE_CALL_AFTER_SLEEP); /* Note that g_pserver->events_processed_while_blocked will also get * incremeted by callbacks called by the event loop handlers. */ - g_pserver->events_processed_while_blocked += ae_events; - long long events = g_pserver->events_processed_while_blocked - startval; + eventsCount += ae_events; + long long events = eventsCount - startval; if (!events) break; } ProcessingEventsWhileBlocked = 0; @@ -3847,6 +3878,8 @@ void processEventsWhileBlocked(int iel) { locker.arm(nullptr); locker.release(); + g_pserver->events_processed_while_blocked += eventsCount; + whileBlockedCron(); // Restore it so the calling code is not confused diff --git a/src/notify.cpp b/src/notify.cpp index 643b2d6aa..fa3133bd1 100644 --- a/src/notify.cpp +++ b/src/notify.cpp @@ -56,6 +56,7 @@ int keyspaceEventsStringToFlags(char *classes) { case 'E': flags |= NOTIFY_KEYEVENT; break; case 't': flags |= NOTIFY_STREAM; break; case 'm': flags |= NOTIFY_KEY_MISS; break; + case 'd': flags |= NOTIFY_MODULE; break; default: return -1; } } @@ -82,6 +83,7 @@ sds keyspaceEventsFlagsToString(int flags) { if (flags & NOTIFY_EXPIRED) res = sdscatlen(res,"x",1); if (flags & NOTIFY_EVICTED) res = sdscatlen(res,"e",1); if (flags & NOTIFY_STREAM) res = sdscatlen(res,"t",1); + if (flags & NOTIFY_MODULE) res = sdscatlen(res,"d",1); } if (flags & NOTIFY_KEYSPACE) res = sdscatlen(res,"K",1); if (flags & NOTIFY_KEYEVENT) res = sdscatlen(res,"E",1); diff --git a/src/object.cpp b/src/object.cpp index 7c858449e..575a05018 100644 --- a/src/object.cpp +++ b/src/object.cpp @@ -793,7 +793,11 @@ int getRangeLongFromObjectOrReply(client *c, robj *o, long min, long max, long * } int getPositiveLongFromObjectOrReply(client *c, robj *o, long *target, const char *msg) { - return getRangeLongFromObjectOrReply(c, o, 0, LONG_MAX, target, msg); + if (msg) { + return getRangeLongFromObjectOrReply(c, o, 0, LONG_MAX, target, msg); + } else { + return getRangeLongFromObjectOrReply(c, o, 0, LONG_MAX, target, "value is out of range, must be positive"); + } } int getIntFromObjectOrReply(client *c, robj *o, int *target, const char *msg) { diff --git a/src/pubsub.cpp b/src/pubsub.cpp index 8e60dc91e..76bf8d22a 100644 --- a/src/pubsub.cpp +++ b/src/pubsub.cpp @@ -349,21 +349,6 @@ int pubsubPublishMessage(robj *channel, robj *message) { return receivers; } -/* This wraps handling ACL channel permissions for the given client. */ -int pubsubCheckACLPermissionsOrReply(client *c, int idx, int count, int literal) { - /* Check if the user can run the command according to the current - * ACLs. */ - int acl_chanpos; - int acl_retval = ACLCheckPubsubPerm(c,idx,count,literal,&acl_chanpos); - if (acl_retval == ACL_DENIED_CHANNEL) { - addACLLogEntry(c,acl_retval,acl_chanpos,NULL); - addReplyError(c, - "-NOPERM this user has no permissions to access " - "one of the channels used as arguments"); - } - return acl_retval; -} - /*----------------------------------------------------------------------------- * Pubsub commands implementation *----------------------------------------------------------------------------*/ @@ -372,7 +357,6 @@ int pubsubCheckACLPermissionsOrReply(client *c, int idx, int count, int literal) void subscribeCommand(client *c) { int j; serverAssert(GlobalLocksAcquired()); - if (pubsubCheckACLPermissionsOrReply(c,1,c->argc-1,0) != ACL_OK) return; if ((c->flags & CLIENT_DENY_BLOCKING) && !(c->flags & CLIENT_MULTI)) { /** * A client that has CLIENT_DENY_BLOCKING flag on @@ -407,7 +391,6 @@ void unsubscribeCommand(client *c) { void psubscribeCommand(client *c) { int j; serverAssert(GlobalLocksAcquired()); - if (pubsubCheckACLPermissionsOrReply(c,1,c->argc-1,1) != ACL_OK) return; if ((c->flags & CLIENT_DENY_BLOCKING) && !(c->flags & CLIENT_MULTI)) { /** * A client that has CLIENT_DENY_BLOCKING flag on @@ -440,7 +423,6 @@ void punsubscribeCommand(client *c) { /* PUBLISH */ void publishCommand(client *c) { - if (pubsubCheckACLPermissionsOrReply(c,1,1,0) != ACL_OK) return; int receivers = pubsubPublishMessage(c->argv[1],c->argv[2]); if (g_pserver->cluster_enabled) clusterPropagatePublish(c->argv[1],c->argv[2]); diff --git a/src/quicklist.c b/src/quicklist.c index b76cde29f..d3890f893 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -315,7 +315,9 @@ REDIS_STATIC void __quicklistCompress(const quicklist *quicklist, if (forward == node || reverse == node) in_depth = 1; - if (forward == reverse) + /* We passed into compress depth of opposite side of the quicklist + * so there's no need to compress anything and we can exit. */ + if (forward == reverse || forward->next == reverse) return; forward = forward->next; @@ -325,11 +327,9 @@ REDIS_STATIC void __quicklistCompress(const quicklist *quicklist, if (!in_depth) quicklistCompressNode(node); - if (depth > 2) { - /* At this point, forward and reverse are one node beyond depth */ - quicklistCompressNode(forward); - quicklistCompressNode(reverse); - } + /* At this point, forward and reverse are one node beyond depth */ + quicklistCompressNode(forward); + quicklistCompressNode(reverse); } #define quicklistCompress(_ql, _node) \ @@ -380,10 +380,11 @@ REDIS_STATIC void __quicklistInsertNode(quicklist *quicklist, quicklist->head = quicklist->tail = new_node; } + /* Update len first, so in __quicklistCompress we know exactly len */ + quicklist->len++; + if (old_node) quicklistCompress(quicklist, old_node); - - quicklist->len++; } /* Wrappers for node inserting around existing node. */ @@ -602,15 +603,16 @@ REDIS_STATIC void __quicklistDelNode(quicklist *quicklist, quicklist->head = node->next; } + /* Update len first, so in __quicklistCompress we know exactly len */ + quicklist->len--; + quicklist->count -= node->count; + /* If we deleted a node within our compress depth, we * now have compressed nodes needing to be decompressed. */ __quicklistCompress(quicklist, NULL); - quicklist->count -= node->count; - zfree(node->zl); zfree(node); - quicklist->len--; } /* Delete one entry from list given the node for the entry and a pointer @@ -1296,17 +1298,24 @@ void quicklistRotate(quicklist *quicklist) { /* First, get the tail entry */ unsigned char *p = ziplistIndex(quicklist->tail->zl, -1); - unsigned char *value; + unsigned char *value, *tmp; long long longval; unsigned int sz; char longstr[32] = {0}; - ziplistGet(p, &value, &sz, &longval); + ziplistGet(p, &tmp, &sz, &longval); /* If value found is NULL, then ziplistGet populated longval instead */ - if (!value) { + if (!tmp) { /* Write the longval as a string so we can re-add it */ sz = ll2string(longstr, sizeof(longstr), longval); value = (unsigned char *)longstr; + } else if (quicklist->len == 1) { + /* Copy buffer since there could be a memory overlap when move + * entity from tail to head in the same ziplist. */ + value = zmalloc(sz, MALLOC_SHARED); + memcpy(value, tmp, sz); + } else { + value = tmp; } /* Add tail entry to head (must happen before tail is deleted). */ @@ -1321,6 +1330,8 @@ void quicklistRotate(quicklist *quicklist) { /* Remove tail entry. */ quicklistDelIndex(quicklist, quicklist->tail, &p); + if (value != (unsigned char*)longstr && value != tmp) + zfree(value); } /* pop from quicklist and return result in 'data' ptr. Value of 'data' @@ -1509,8 +1520,6 @@ void quicklistBookmarksClear(quicklist *ql) { #define yell(str, ...) printf("ERROR! " str "\n\n", __VA_ARGS__) -#define OK printf("\tOK\n") - #define ERROR \ do { \ printf("\tERROR!\n"); \ @@ -1630,7 +1639,6 @@ static int _ql_verify(quicklist *ql, uint32_t len, uint32_t count, } if (ql->len == 0 && !errors) { - OK; return errors; } @@ -1679,8 +1687,6 @@ static int _ql_verify(quicklist *ql, uint32_t len, uint32_t count, } } - if (!errors) - OK; return errors; } @@ -1692,9 +1698,10 @@ static char *genstr(char *prefix, int i) { } /* main test, but callable from other files */ -int quicklistTest(int argc, char *argv[]) { +int quicklistTest(int argc, char *argv[], int accurate) { UNUSED(argc); UNUSED(argv); + UNUSED(accurate); unsigned int err = 0; int optimize_start = @@ -1703,11 +1710,14 @@ int quicklistTest(int argc, char *argv[]) { printf("Starting optimization offset at: %d\n", optimize_start); int options[] = {0, 1, 2, 3, 4, 5, 6, 10}; + int fills[] = {-5, -4, -3, -2, -1, 0, + 1, 2, 32, 66, 128, 999}; size_t option_count = sizeof(options) / sizeof(*options); + int fill_count = (int)(sizeof(fills) / sizeof(*fills)); long long runtime[option_count]; for (int _i = 0; _i < (int)option_count; _i++) { - printf("Testing Option %d\n", options[_i]); + printf("Testing Compression option %d\n", options[_i]); long long start = mstime(); TEST("create list") { @@ -1732,57 +1742,53 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(ql); } - for (int f = optimize_start; f < 32; f++) { - TEST_DESC("add to tail 5x at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("add to tail 5x at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 5; i++) quicklistPushTail(ql, genstr("hello", i), 32); if (ql->count != 5) ERROR; - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 1, 5, 5, 5); quicklistRelease(ql); } } - for (int f = optimize_start; f < 32; f++) { - TEST_DESC("add to head 5x at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("add to head 5x at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 5; i++) quicklistPushHead(ql, genstr("hello", i), 32); if (ql->count != 5) ERROR; - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 1, 5, 5, 5); quicklistRelease(ql); } } - for (int f = optimize_start; f < 512; f++) { - TEST_DESC("add to tail 500x at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("add to tail 500x at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 500; i++) quicklistPushTail(ql, genstr("hello", i), 64); if (ql->count != 500) ERROR; - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 16, 500, 32, 20); quicklistRelease(ql); } } - for (int f = optimize_start; f < 512; f++) { - TEST_DESC("add to head 500x at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("add to head 500x at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 500; i++) quicklistPushHead(ql, genstr("hello", i), 32); if (ql->count != 500) ERROR; - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 16, 500, 20, 32); quicklistRelease(ql); } @@ -1795,9 +1801,9 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(ql); } - for (int f = optimize_start; f < 32; f++) { - TEST("rotate one val once") { - quicklist *ql = quicklistNew(f, options[_i]); + TEST("rotate one val once") { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); quicklistPushHead(ql, "hello", 6); quicklistRotate(ql); /* Ignore compression verify because ziplist is @@ -1807,10 +1813,9 @@ int quicklistTest(int argc, char *argv[]) { } } - for (int f = optimize_start; f < 3; f++) { - TEST_DESC("rotate 500 val 5000 times at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("rotate 500 val 5000 times at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); quicklistPushHead(ql, "900", 3); quicklistPushHead(ql, "7000", 4); quicklistPushHead(ql, "-1200", 5); @@ -1822,11 +1827,11 @@ int quicklistTest(int argc, char *argv[]) { ql_info(ql); quicklistRotate(ql); } - if (f == 1) + if (fills[f] == 1) ql_verify(ql, 504, 504, 1, 1); - else if (f == 2) + else if (fills[f] == 2) ql_verify(ql, 252, 504, 2, 2); - else if (f == 32) + else if (fills[f] == 32) ql_verify(ql, 16, 504, 32, 24); quicklistRelease(ql); } @@ -2003,11 +2008,10 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(ql); } - for (int f = optimize_start; f < 12; f++) { - TEST_DESC("insert once in elements while iterating at fill %d at " - "compress %d\n", - f, options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("insert once in elements while iterating at compress %d", + options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); quicklistPushTail(ql, "abc", 3); quicklistSetFill(ql, 1); quicklistPushTail(ql, "def", 3); /* force to unique node */ @@ -2059,12 +2063,10 @@ int quicklistTest(int argc, char *argv[]) { } } - for (int f = optimize_start; f < 1024; f++) { - TEST_DESC( - "insert [before] 250 new in middle of 500 elements at fill" - " %d at compress %d", - f, options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("insert [before] 250 new in middle of 500 elements at compress %d", + options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 500; i++) quicklistPushTail(ql, genstr("hello", i), 32); for (int i = 0; i < 250; i++) { @@ -2072,17 +2074,16 @@ int quicklistTest(int argc, char *argv[]) { quicklistIndex(ql, 250, &entry); quicklistInsertBefore(ql, &entry, genstr("abc", i), 32); } - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 25, 750, 32, 20); quicklistRelease(ql); } } - for (int f = optimize_start; f < 1024; f++) { - TEST_DESC("insert [after] 250 new in middle of 500 elements at " - "fill %d at compress %d", - f, options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("insert [after] 250 new in middle of 500 elements at compress %d", + options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 500; i++) quicklistPushHead(ql, genstr("hello", i), 32); for (int i = 0; i < 250; i++) { @@ -2094,7 +2095,7 @@ int quicklistTest(int argc, char *argv[]) { if (ql->count != 750) ERR("List size not 750, but rather %ld", ql->count); - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 26, 750, 20, 32); quicklistRelease(ql); } @@ -2132,70 +2133,58 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(copy); } - for (int f = optimize_start; f < 512; f++) { + for (int f = 0; f < fill_count; f++) { TEST_DESC("index 1,200 from 500 list at fill %d at compress %d", f, options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 500; i++) quicklistPushTail(ql, genstr("hello", i + 1), 32); quicklistEntry entry; quicklistIndex(ql, 1, &entry); - if (!strcmp((char *)entry.value, "hello2")) - OK; - else + if (strcmp((char *)entry.value, "hello2") != 0) ERR("Value: %s", entry.value); quicklistIndex(ql, 200, &entry); - if (!strcmp((char *)entry.value, "hello201")) - OK; - else + if (strcmp((char *)entry.value, "hello201") != 0) ERR("Value: %s", entry.value); quicklistRelease(ql); } - TEST_DESC("index -1,-2 from 500 list at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("index -1,-2 from 500 list at fill %d at compress %d", + fills[f], options[_i]) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 500; i++) quicklistPushTail(ql, genstr("hello", i + 1), 32); quicklistEntry entry; quicklistIndex(ql, -1, &entry); - if (!strcmp((char *)entry.value, "hello500")) - OK; - else + if (strcmp((char *)entry.value, "hello500") != 0) ERR("Value: %s", entry.value); quicklistIndex(ql, -2, &entry); - if (!strcmp((char *)entry.value, "hello499")) - OK; - else + if (strcmp((char *)entry.value, "hello499") != 0) ERR("Value: %s", entry.value); quicklistRelease(ql); } - TEST_DESC("index -100 from 500 list at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("index -100 from 500 list at fill %d at compress %d", + fills[f], options[_i]) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 500; i++) quicklistPushTail(ql, genstr("hello", i + 1), 32); quicklistEntry entry; quicklistIndex(ql, -100, &entry); - if (!strcmp((char *)entry.value, "hello401")) - OK; - else + if (strcmp((char *)entry.value, "hello401") != 0) ERR("Value: %s", entry.value); quicklistRelease(ql); } TEST_DESC("index too big +1 from 50 list at fill %d at compress %d", - f, options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + fills[f], options[_i]) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 50; i++) quicklistPushTail(ql, genstr("hello", i + 1), 32); quicklistEntry entry; if (quicklistIndex(ql, 50, &entry)) ERR("Index found at 50 with 50 list: %.*s", entry.sz, entry.value); - else - OK; quicklistRelease(ql); } } @@ -2367,12 +2356,11 @@ int quicklistTest(int argc, char *argv[]) { quicklistReplaceAtIndex(ql, 1, "foo", 3); quicklistReplaceAtIndex(ql, -1, "bar", 3); quicklistRelease(ql); - OK; } - for (int f = optimize_start; f < 16; f++) { - TEST_DESC("lrem test at fill %d at compress %d", f, options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("lrem test at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); char *words[] = {"abc", "foo", "bar", "foobar", "foobared", "zap", "bar", "test", "foo"}; char *result[] = {"abc", "foo", "foobar", "foobared", @@ -2397,14 +2385,12 @@ int quicklistTest(int argc, char *argv[]) { /* check result of lrem 0 bar */ iter = quicklistGetIterator(ql, AL_START_HEAD); i = 0; - int ok = 1; while (quicklistNext(iter, &entry)) { /* Result must be: abc, foo, foobar, foobared, zap, test, * foo */ if (strncmp((char *)entry.value, result[i], entry.sz)) { ERR("No match at position %d, got %.*s instead of %s", i, entry.sz, entry.value, result[i]); - ok = 0; } i++; } @@ -2441,23 +2427,18 @@ int quicklistTest(int argc, char *argv[]) { entry.sz)) { ERR("No match at position %d, got %.*s instead of %s", i, entry.sz, entry.value, resultB[resB - 1 - i]); - ok = 0; } i++; } quicklistReleaseIterator(iter); - /* final result of all tests */ - if (ok) - OK; quicklistRelease(ql); } } - for (int f = optimize_start; f < 16; f++) { - TEST_DESC("iterate reverse + delete at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("iterate reverse + delete at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); quicklistPushTail(ql, "abc", 3); quicklistPushTail(ql, "def", 3); quicklistPushTail(ql, "hij", 3); @@ -2494,10 +2475,9 @@ int quicklistTest(int argc, char *argv[]) { } } - for (int f = optimize_start; f < 800; f++) { - TEST_DESC("iterator at index test at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("iterator at index test at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); char num[32]; long long nums[5000]; for (int i = 0; i < 760; i++) { @@ -2521,10 +2501,9 @@ int quicklistTest(int argc, char *argv[]) { } } - for (int f = optimize_start; f < 40; f++) { - TEST_DESC("ltrim test A at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("ltrim test A at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); char num[32]; long long nums[5000]; for (int i = 0; i < 32; i++) { @@ -2532,7 +2511,7 @@ int quicklistTest(int argc, char *argv[]) { int sz = ll2string(num, sizeof(num), nums[i]); quicklistPushTail(ql, num, sz); } - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 1, 32, 32, 32); /* ltrim 25 53 (keep [25,32] inclusive = 7 remaining) */ quicklistDelRange(ql, 0, 25); @@ -2545,18 +2524,17 @@ int quicklistTest(int argc, char *argv[]) { "%lld", entry.longval, nums[25 + i]); } - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 1, 7, 7, 7); quicklistRelease(ql); } } - for (int f = optimize_start; f < 40; f++) { - TEST_DESC("ltrim test B at fill %d at compress %d", f, - options[_i]) { + TEST_DESC("ltrim test B at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { /* Force-disable compression because our 33 sequential * integers don't compress and the check always fails. */ - quicklist *ql = quicklistNew(f, QUICKLIST_NOCOMPRESS); + quicklist *ql = quicklistNew(fills[f], QUICKLIST_NOCOMPRESS); char num[32]; long long nums[5000]; for (int i = 0; i < 33; i++) { @@ -2564,24 +2542,20 @@ int quicklistTest(int argc, char *argv[]) { int sz = ll2string(num, sizeof(num), nums[i]); quicklistPushTail(ql, num, sz); } - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 2, 33, 32, 1); /* ltrim 5 16 (keep [5,16] inclusive = 12 remaining) */ quicklistDelRange(ql, 0, 5); quicklistDelRange(ql, -16, 16); - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 1, 12, 12, 12); quicklistEntry entry; quicklistIndex(ql, 0, &entry); if (entry.longval != 5) ERR("A: longval not 5, but %lld", entry.longval); - else - OK; quicklistIndex(ql, -1, &entry); if (entry.longval != 16) ERR("B! got instead: %lld", entry.longval); - else - OK; quicklistPushTail(ql, "bobobob", 7); quicklistIndex(ql, -1, &entry); if (strncmp((char *)entry.value, "bobobob", 7)) @@ -2598,10 +2572,9 @@ int quicklistTest(int argc, char *argv[]) { } } - for (int f = optimize_start; f < 40; f++) { - TEST_DESC("ltrim test C at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("ltrim test C at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); char num[32]; long long nums[5000]; for (int i = 0; i < 33; i++) { @@ -2609,28 +2582,25 @@ int quicklistTest(int argc, char *argv[]) { int sz = ll2string(num, sizeof(num), nums[i]); quicklistPushTail(ql, num, sz); } - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 2, 33, 32, 1); /* ltrim 3 3 (keep [3,3] inclusive = 1 remaining) */ quicklistDelRange(ql, 0, 3); quicklistDelRange(ql, -29, 4000); /* make sure not loop forever */ - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 1, 1, 1, 1); quicklistEntry entry; quicklistIndex(ql, 0, &entry); if (entry.longval != -5157318210846258173) ERROR; - else - OK; quicklistRelease(ql); } } - for (int f = optimize_start; f < 40; f++) { - TEST_DESC("ltrim test D at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("ltrim test D at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); char num[32]; long long nums[5000]; for (int i = 0; i < 33; i++) { @@ -2638,7 +2608,7 @@ int quicklistTest(int argc, char *argv[]) { int sz = ll2string(num, sizeof(num), nums[i]); quicklistPushTail(ql, num, sz); } - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 2, 33, 32, 1); quicklistDelRange(ql, -12, 3); if (ql->count != 30) @@ -2648,9 +2618,8 @@ int quicklistTest(int argc, char *argv[]) { } } - for (int f = optimize_start; f < 72; f++) { - TEST_DESC("create quicklist from ziplist at fill %d at compress %d", - f, options[_i]) { + TEST_DESC("create quicklist from ziplist at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { unsigned char *zl = ziplistNew(); long long nums[64]; char num[64]; @@ -2664,12 +2633,12 @@ int quicklistTest(int argc, char *argv[]) { zl = ziplistPush(zl, (unsigned char *)genstr("hello", i), 32, ZIPLIST_TAIL); } - quicklist *ql = quicklistCreateFromZiplist(f, options[_i], zl); - if (f == 1) + quicklist *ql = quicklistCreateFromZiplist(fills[f], options[_i], zl); + if (fills[f] == 1) ql_verify(ql, 66, 66, 1, 1); - else if (f == 32) + else if (fills[f] == 32) ql_verify(ql, 3, 66, 32, 2); - else if (f == 66) + else if (fills[f] == 66) ql_verify(ql, 1, 66, 66, 66); quicklistRelease(ql); } @@ -2682,45 +2651,56 @@ int quicklistTest(int argc, char *argv[]) { /* Run a longer test of compression depth outside of primary test loop. */ int list_sizes[] = {250, 251, 500, 999, 1000}; long long start = mstime(); - for (int list = 0; list < (int)(sizeof(list_sizes) / sizeof(*list_sizes)); - list++) { - for (int f = optimize_start; f < 128; f++) { - for (int depth = 1; depth < 40; depth++) { - /* skip over many redundant test cases */ - TEST_DESC("verify specific compression of interior nodes with " - "%d list " - "at fill %d at compress %d", - list_sizes[list], f, depth) { - quicklist *ql = quicklistNew(f, depth); + int list_count = accurate ? (int)(sizeof(list_sizes) / sizeof(*list_sizes)) : 1; + for (int list = 0; list < list_count; list++) { + TEST_DESC("verify specific compression of interior nodes with %d list ", + list_sizes[list]) { + for (int f = 0; f < fill_count; f++) { + for (int depth = 1; depth < 40; depth++) { + /* skip over many redundant test cases */ + quicklist *ql = quicklistNew(fills[f], depth); for (int i = 0; i < list_sizes[list]; i++) { quicklistPushTail(ql, genstr("hello TAIL", i + 1), 64); quicklistPushHead(ql, genstr("hello HEAD", i + 1), 64); } - quicklistNode *node = ql->head; - unsigned int low_raw = ql->compress; - unsigned int high_raw = ql->len - ql->compress; - - for (unsigned int at = 0; at < ql->len; - at++, node = node->next) { - if (at < low_raw || at >= high_raw) { - if (node->encoding != QUICKLIST_NODE_ENCODING_RAW) { - ERR("Incorrect compression: node %d is " - "compressed at depth %d ((%u, %u); total " - "nodes: %lu; size: %u)", - at, depth, low_raw, high_raw, ql->len, - node->sz); + for (int step = 0; step < 2; step++) { + /* test remove node */ + if (step == 1) { + for (int i = 0; i < list_sizes[list] / 2; i++) { + unsigned char *data; + quicklistPop(ql, QUICKLIST_HEAD, &data, NULL, NULL); + zfree(data); + quicklistPop(ql, QUICKLIST_TAIL, &data, NULL, NULL); + zfree(data); } - } else { - if (node->encoding != QUICKLIST_NODE_ENCODING_LZF) { - ERR("Incorrect non-compression: node %d is NOT " - "compressed at depth %d ((%u, %u); total " - "nodes: %lu; size: %u; attempted: %d)", - at, depth, low_raw, high_raw, ql->len, - node->sz, node->attempted_compress); + } + quicklistNode *node = ql->head; + unsigned int low_raw = ql->compress; + unsigned int high_raw = ql->len - ql->compress; + + for (unsigned int at = 0; at < ql->len; + at++, node = node->next) { + if (at < low_raw || at >= high_raw) { + if (node->encoding != QUICKLIST_NODE_ENCODING_RAW) { + ERR("Incorrect compression: node %d is " + "compressed at depth %d ((%u, %u); total " + "nodes: %lu; size: %u)", + at, depth, low_raw, high_raw, ql->len, + node->sz); + } + } else { + if (node->encoding != QUICKLIST_NODE_ENCODING_LZF) { + ERR("Incorrect non-compression: node %d is NOT " + "compressed at depth %d ((%u, %u); total " + "nodes: %lu; size: %u; attempted: %d)", + at, depth, low_raw, high_raw, ql->len, + node->sz, node->attempted_compress); + } } } } + quicklistRelease(ql); } } diff --git a/src/quicklist.h b/src/quicklist.h index 38db36f5b..318077eb4 100644 --- a/src/quicklist.h +++ b/src/quicklist.h @@ -213,7 +213,7 @@ quicklistNode *quicklistBookmarkFind(quicklist *ql, const char *name); void quicklistBookmarksClear(quicklist *ql); #ifdef REDIS_TEST -int quicklistTest(int argc, char *argv[]); +int quicklistTest(int argc, char *argv[], int accurate); #endif /* Directions for iterators */ diff --git a/src/rdb.cpp b/src/rdb.cpp index 4de651481..6edf4231e 100644 --- a/src/rdb.cpp +++ b/src/rdb.cpp @@ -1118,8 +1118,7 @@ size_t rdbSavedObjectLen(robj *o, robj *key) { /* Save a key-value pair, with expire time, type, key, value. * On error -1 is returned. - * On success if the key was actually saved 1 is returned, otherwise 0 - * is returned (the key was already expired). */ + * On success if the key was actually saved 1 is returned. */ int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, expireEntry *pexpire) { int savelru = g_pserver->maxmemory_policy & MAXMEMORY_FLAG_LRU; int savelfu = g_pserver->maxmemory_policy & MAXMEMORY_FLAG_LFU; @@ -2261,16 +2260,17 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) { return NULL; } moduleType *mt = moduleTypeLookupModuleByID(moduleid); - char name[10]; if (rdbCheckMode && rdbtype == RDB_TYPE_MODULE_2) { + char name[10]; moduleTypeNameByID(name,moduleid); return rdbLoadCheckModuleValue(rdb,name); } if (mt == NULL) { + char name[10]; moduleTypeNameByID(name,moduleid); - rdbReportCorruptRDB("The RDB file contains module data I can't load: no matching module '%s'", name); + rdbReportCorruptRDB("The RDB file contains module data I can't load: no matching module type '%s'", name); return NULL; } RedisModuleIO io; @@ -2297,7 +2297,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) { return NULL; } if (eof != RDB_MODULE_OPCODE_EOF) { - rdbReportCorruptRDB("The RDB file contains module data for the module '%s' that is not terminated by the proper module value EOF marker", name); + rdbReportCorruptRDB("The RDB file contains module data for the module '%s' that is not terminated by " + "the proper module value EOF marker", moduleTypeModuleName(mt)); if (ptr) { o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */ decrRefCount(o); @@ -2307,8 +2308,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) { } if (ptr == NULL) { - moduleTypeNameByID(name,moduleid); - rdbReportCorruptRDB("The RDB file contains module data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name); + rdbReportCorruptRDB("The RDB file contains module data for the module type '%s', that the responsible " + "module is not able to load. Check for modules log above for additional clues.", + moduleTypeModuleName(mt)); return NULL; } o = createModuleObject(mt,ptr); @@ -2935,7 +2937,7 @@ void backgroundSaveDoneHandler(int exitcode, int bysignal) { * the cleanup needed. */ void killRDBChild(void) { kill(g_pserver->child_pid, SIGUSR1); - /* Because we are not using here wait4 (like we have in killAppendOnlyChild + /* Because we are not using here waitpid (like we have in killAppendOnlyChild * and TerminateModuleForkChild), all the cleanup operations is done by * checkChildrenDone, that later will find that the process killed. * This includes: diff --git a/src/redis-benchmark.cpp b/src/redis-benchmark.cpp index 39cb7c648..d243fe6ca 100644 --- a/src/redis-benchmark.cpp +++ b/src/redis-benchmark.cpp @@ -106,7 +106,6 @@ static struct config { int showerrors; long long start; long long totlatency; - long long *latency; const char *title; list *clients; int quiet; @@ -1661,7 +1660,10 @@ int showThroughput(struct aeEventLoop *eventLoop, long long id, void *clientData const float instantaneous_rps = (float)(requests_finished-previous_requests_finished)/instantaneous_dt; config.previous_tick = current_tick; atomicSet(config.previous_requests_finished,requests_finished); - config.last_printed_bytes = printf("%s: rps=%.1f (overall: %.1f) avg_msec=%.3f (overall: %.3f)\r", config.title, instantaneous_rps, rps, hdr_mean(config.current_sec_latency_histogram)/1000.0f, hdr_mean(config.latency_histogram)/1000.0f); + int printed_bytes = printf("%s: rps=%.1f (overall: %.1f) avg_msec=%.3f (overall: %.3f)\r", config.title, instantaneous_rps, rps, hdr_mean(config.current_sec_latency_histogram)/1000.0f, hdr_mean(config.latency_histogram)/1000.0f); + if (printed_bytes > config.last_printed_bytes){ + config.last_printed_bytes = printed_bytes; + } hdr_reset(config.current_sec_latency_histogram); fflush(stdout); return 250; /* every 250ms */ diff --git a/src/redis-check-rdb.cpp b/src/redis-check-rdb.cpp index 2b1471e15..1dbf47fbd 100644 --- a/src/redis-check-rdb.cpp +++ b/src/redis-check-rdb.cpp @@ -192,6 +192,7 @@ int redis_check_rdb(const char *rdbfilename, FILE *fp) { int closefile = (fp == NULL); if (fp == NULL && (fp = fopen(rdbfilename,"r")) == NULL) return 1; + startLoadingFile(fp, rdbfilename, RDBFLAGS_NONE); rioInitWithFile(&rdb,fp); rdbstate.rio = &rdb; rdb.update_cksum = rdbLoadProgressCallback; @@ -208,7 +209,6 @@ int redis_check_rdb(const char *rdbfilename, FILE *fp) { } expiretime = -1; - startLoadingFile(fp, rdbfilename, RDBFLAGS_NONE); while(1) { robj *key, *val; diff --git a/src/redis-cli-cpphelper.cpp b/src/redis-cli-cpphelper.cpp index e9269cf28..8ea27429e 100644 --- a/src/redis-cli-cpphelper.cpp +++ b/src/redis-cli-cpphelper.cpp @@ -397,8 +397,12 @@ int clusterManagerGetAntiAffinityScore(clusterManagerNodeArray *ipnodes, else types = sdsempty(); /* Master type 'm' is always set as the first character of the * types string. */ - if (!node->replicate) types = sdscatprintf(types, "m%s", types); - else types = sdscat(types, "s"); + if (node->replicate) types = sdscat(types, "s"); + else { + sds s = sdscatsds(sdsnew("m"), types); + sdsfree(types); + types = s; + } dictReplace(related, key, types); } /* Now it's trivial to check, for each related group having the diff --git a/src/redis-cli.c b/src/redis-cli.c index ca9796b9b..95fc56f3d 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -238,15 +238,17 @@ static void parseRedisUri(const char *uri) { if (!strncasecmp(tlsscheme, curr, strlen(tlsscheme))) { #ifdef USE_OPENSSL config.tls = 1; + curr += strlen(tlsscheme); #else fprintf(stderr,"rediss:// is only supported when redis-cli is compiled with OpenSSL\n"); exit(1); #endif - } else if (strncasecmp(scheme, curr, strlen(scheme))) { + } else if (!strncasecmp(scheme, curr, strlen(scheme))) { + curr += strlen(scheme); + } else { fprintf(stderr,"Invalid URI scheme\n"); exit(1); } - curr += strlen(scheme); if (curr == end) return; /* Extract user info. */ @@ -568,6 +570,23 @@ static void freeHintsCallback(void *ptr) { * Networking / parsing *--------------------------------------------------------------------------- */ +/* Unquote a null-terminated string and return it as a binary-safe sds. */ +static sds unquoteCString(char *str) { + int count; + sds *unquoted = sdssplitargs(str, &count); + sds res = NULL; + + if (unquoted && count == 1) { + res = unquoted[0]; + unquoted[0] = NULL; + } + + if (unquoted) + sdsfreesplitres(unquoted, count); + + return res; +} + /* Send AUTH command to the server */ static int cliAuth(redisContext *ctx, char *user, char *auth) { redisReply *reply; @@ -1338,6 +1357,8 @@ static int parseOptions(int argc, char **argv) { config.output = OUTPUT_RAW; } else if (!strcmp(argv[i],"--no-raw")) { config.output = OUTPUT_STANDARD; + } else if (!strcmp(argv[i],"--quoted-input")) { + config.quoted_input = 1; } else if (!strcmp(argv[i],"--csv")) { config.output = OUTPUT_CSV; } else if (!strcmp(argv[i],"--latency")) { @@ -1362,7 +1383,15 @@ static int parseOptions(int argc, char **argv) { } else if (!strcmp(argv[i],"--scan")) { config.scan_mode = 1; } else if (!strcmp(argv[i],"--pattern") && !lastarg) { - config.pattern = argv[++i]; + sdsfree(config.pattern); + config.pattern = sdsnew(argv[++i]); + } else if (!strcmp(argv[i],"--quoted-pattern") && !lastarg) { + sdsfree(config.pattern); + config.pattern = unquoteCString(argv[++i]); + if (!config.pattern) { + fprintf(stderr,"Invalid quoted string specified for --quoted-pattern.\n"); + exit(1); + } } else if (!strcmp(argv[i],"--intrinsic-latency") && !lastarg) { config.intrinsic_latency_mode = 1; config.intrinsic_latency_duration = atoi(argv[++i]); @@ -1648,6 +1677,7 @@ static void usage(void) { " --raw Use raw formatting for replies (default when STDOUT is\n" " not a tty).\n" " --no-raw Force formatted output even when STDOUT is not a tty.\n" +" --quoted-input Force input to be handled as quoted strings.\n" " --csv Output in CSV format.\n" " --show-pushes Whether to print RESP3 PUSH messages. Enabled by default when\n" " STDOUT is a tty but can be overriden with --show-pushes no.\n" @@ -1683,6 +1713,8 @@ static void usage(void) { " --scan List all keys using the SCAN command.\n" " --pattern Keys pattern when using the --scan, --bigkeys or --hotkeys\n" " options (default: *).\n" +" --quoted-pattern Same as --pattern, but the specified string can be\n" +" quoted, in order to pass an otherwise non binary-safe string.\n" " --intrinsic-latency Run a test to measure intrinsic system latency.\n" " The test will run for the specified amount of seconds.\n" " --eval Send an EVAL command using the Lua script at .\n" @@ -1708,6 +1740,7 @@ static void usage(void) { " keydb-cli get mypasswd\n" " keydb-cli -r 100 lpush mylist x\n" " keydb-cli -r 100 -i 1 info | grep used_memory_human:\n" +" keydb-cli --quoted-input set '\"null-\\x00-separated\"' value\n" " keydb-cli --eval myscript.lua key1 key2 , arg1 arg2 arg3\n" " keydb-cli --scan --pattern '*:12345*'\n" "\n" @@ -1737,22 +1770,28 @@ int confirmWithYes(const char *msg, int ignore_force) { return (nread != 0 && !strcmp("yes", buf)); } -/* Turn the plain C strings into Sds strings */ -static char **convertToSds(int count, char** args) { - int j; - char **sds = zmalloc(sizeof(char*)*count, MALLOC_LOCAL); +/* Create an sds array from argv, either as-is or by dequoting every + * element. When quoted is non-zero, may return a NULL to indicate an + * invalid quoted string. + */ +static sds *getSdsArrayFromArgv(int argc, char **argv, int quoted) { + sds *res = sds_malloc(sizeof(sds) * argc); - for(j = 0; j < count; j++) - sds[j] = sdsnew(args[j]); + for (int j = 0; j < argc; j++) { + if (quoted) { + sds unquoted = unquoteCString(argv[j]); + if (!unquoted) { + while (--j >= 0) sdsfree(res[j]); + sds_free(res); + return NULL; + } + res[j] = unquoted; + } else { + res[j] = sdsnew(argv[j]); + } + } - return sds; -} - -static void freeConvertedSds(int count, char **sds) { - int j; - for (j = 0; j < count; j++) - sdsfree(sds[j]); - zfree(sds); + return res; } static int issueCommandRepeat(int argc, char **argv, long repeat) { @@ -1985,17 +2024,19 @@ static void repl(void) { static int noninteractive(int argc, char **argv) { int retval = 0; - - argv = convertToSds(argc, argv); - if (config.stdinarg) { - argv = zrealloc(argv, (argc+1)*sizeof(char*), MALLOC_LOCAL); - argv[argc] = readArgFromStdin(); - retval = issueCommand(argc+1, argv); - sdsfree(argv[argc]); - } else { - retval = issueCommand(argc, argv); + sds *sds_args = getSdsArrayFromArgv(argc, argv, config.quoted_input); + if (!sds_args) { + printf("Invalid quoted string\n"); + return 1; } - freeConvertedSds(argc, argv); + if (config.stdinarg) { + sds_args = sds_realloc(sds_args, (argc + 1) * sizeof(sds)); + sds_args[argc] = readArgFromStdin(); + argc++; + } + + retval = issueCommand(argc, sds_args); + sdsfreesplitres(sds_args, argc); return retval; } @@ -6234,7 +6275,10 @@ static void getRDB(clusterManagerNode *node) { redisFree(s); /* Close the connection ASAP as fsync() may take time. */ if (node) node->context = NULL; - fsync(fd); + if (fsync(fd) == -1) { + fprintf(stderr,"Fail to fsync '%s': %s\n", filename, strerror(errno)); + exit(1); + } close(fd); if (node) { sdsfree(filename); @@ -6410,8 +6454,8 @@ redisReply *sendScan(unsigned long long *it) { redisReply *reply; if (config.pattern) - reply = redisCommand(context,"SCAN %llu MATCH %s", - *it,config.pattern); + reply = redisCommand(context, "SCAN %llu MATCH %b", + *it, config.pattern, sdslen(config.pattern)); else reply = redisCommand(context,"SCAN %llu",*it); @@ -6446,8 +6490,14 @@ int getDbSize(void) { reply = redisCommand(context, "DBSIZE"); - if(reply == NULL || reply->type != REDIS_REPLY_INTEGER) { - fprintf(stderr, "Couldn't determine DBSIZE!\n"); + if (reply == NULL) { + fprintf(stderr, "\nI/O error\n"); + exit(1); + } else if (reply->type == REDIS_REPLY_ERROR) { + fprintf(stderr, "Couldn't determine DBSIZE: %s\n", reply->str); + exit(1); + } else if (reply->type != REDIS_REPLY_INTEGER) { + fprintf(stderr, "Non INTEGER response from DBSIZE!\n"); exit(1); } @@ -6819,23 +6869,16 @@ static void scanMode(void) { unsigned long long cur = 0; do { - if (config.pattern) - reply = redisCommand(context,"SCAN %llu MATCH %s", - cur,config.pattern); - else - reply = redisCommand(context,"SCAN %llu",cur); - if (reply == NULL) { - printf("I/O error\n"); - exit(1); - } else if (reply->type == REDIS_REPLY_ERROR) { - printf("ERROR: %s\n", reply->str); - exit(1); - } else { - unsigned int j; - - cur = strtoull(reply->element[0]->str,NULL,10); - for (j = 0; j < reply->element[1]->elements; j++) + reply = sendScan(&cur); + for (unsigned int j = 0; j < reply->element[1]->elements; j++) { + if (config.output == OUTPUT_STANDARD) { + sds out = sdscatrepr(sdsempty(), reply->element[1]->element[j]->str, + reply->element[1]->element[j]->len); + printf("%s\n", out); + sdsfree(out); + } else { printf("%s\n", reply->element[1]->element[j]->str); + } } freeReplyObject(reply); } while(cur != 0); diff --git a/src/redis-cli.h b/src/redis-cli.h index 834904c7d..062f74301 100644 --- a/src/redis-cli.h +++ b/src/redis-cli.h @@ -166,7 +166,7 @@ extern struct config { int scan_mode; int intrinsic_latency_mode; int intrinsic_latency_duration; - char *pattern; + sds pattern; char *rdb_filename; int bigkeys; int memkeys; @@ -195,6 +195,7 @@ extern struct config { int disable_motd; int in_multi; int pre_multi_dbnum; + int quoted_input; /* Force input args to be treated as quoted strings */ } config; struct clusterManager { diff --git a/src/redismodule.h b/src/redismodule.h index f9161c3c9..fd29d1d03 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -164,13 +164,14 @@ This flag should not be used directly by the module. #define REDISMODULE_NOTIFY_STREAM (1<<10) /* t */ #define REDISMODULE_NOTIFY_KEY_MISS (1<<11) /* m (Note: This one is excluded from REDISMODULE_NOTIFY_ALL on purpose) */ #define REDISMODULE_NOTIFY_LOADED (1<<12) /* module only key space notification, indicate a key loaded from rdb */ +#define REDISMODULE_NOTIFY_MODULE (1<<13) /* d, module key space notification */ /* Next notification flag, must be updated when adding new flags above! This flag should not be used directly by the module. * Use RedisModule_GetKeyspaceNotificationFlagsAll instead. */ -#define _REDISMODULE_NOTIFY_NEXT (1<<13) +#define _REDISMODULE_NOTIFY_NEXT (1<<14) -#define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM) /* A */ +#define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM | REDISMODULE_NOTIFY_MODULE) /* A */ /* A special pointer that we can use between the core and the module to signal * field deletion, and that is impossible to be a valid pointer. */ @@ -197,6 +198,12 @@ This flag should not be used directly by the module. #define REDISMODULE_NOT_USED(V) ((void) V) +/* Logging level strings */ +#define REDISMODULE_LOGLEVEL_DEBUG "debug" +#define REDISMODULE_LOGLEVEL_VERBOSE "verbose" +#define REDISMODULE_LOGLEVEL_NOTICE "notice" +#define REDISMODULE_LOGLEVEL_WARNING "warning" + /* Bit flags for aux_save_triggers and the aux_load and aux_save callbacks */ #define REDISMODULE_AUX_BEFORE_RDB (1<<0) #define REDISMODULE_AUX_AFTER_RDB (1<<1) @@ -639,6 +646,8 @@ REDISMODULE_API char * (*RedisModule_StringDMA)(RedisModuleKey *key, size_t *len REDISMODULE_API int (*RedisModule_StringTruncate)(RedisModuleKey *key, size_t newlen) REDISMODULE_ATTR; REDISMODULE_API mstime_t (*RedisModule_GetExpire)(RedisModuleKey *key) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_SetExpire)(RedisModuleKey *key, mstime_t expire) REDISMODULE_ATTR; +REDISMODULE_API mstime_t (*RedisModule_GetAbsExpire)(RedisModuleKey *key) REDISMODULE_ATTR; +REDISMODULE_API int (*RedisModule_SetAbsExpire)(RedisModuleKey *key, mstime_t expire) REDISMODULE_ATTR; REDISMODULE_API void (*RedisModule_ResetDataset)(int restart_aof, int async) REDISMODULE_ATTR; REDISMODULE_API unsigned long long (*RedisModule_DbSize)(RedisModuleCtx *ctx) REDISMODULE_ATTR; REDISMODULE_API RedisModuleString * (*RedisModule_RandomKey)(RedisModuleCtx *ctx) REDISMODULE_ATTR; @@ -839,7 +848,7 @@ REDISMODULE_API int (*RedisModule_DefragCursorSet)(RedisModuleDefragCtx *ctx, un REDISMODULE_API int (*RedisModule_DefragCursorGet)(RedisModuleDefragCtx *ctx, unsigned long *cursor) REDISMODULE_ATTR; #endif -#define RedisModule_IsAOFClient(id) ((id) == CLIENT_ID_AOF) +#define RedisModule_IsAOFClient(id) ((id) == UINT64_MAX) /* This is included inline inside each Redis module. */ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) REDISMODULE_ATTR_UNUSED; @@ -911,6 +920,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(StringTruncate); REDISMODULE_GET_API(GetExpire); REDISMODULE_GET_API(SetExpire); + REDISMODULE_GET_API(GetAbsExpire); + REDISMODULE_GET_API(SetAbsExpire); REDISMODULE_GET_API(ResetDataset); REDISMODULE_GET_API(DbSize); REDISMODULE_GET_API(RandomKey); diff --git a/src/replication.cpp b/src/replication.cpp index 3a785e0c8..e6c6ebede 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -1191,15 +1191,32 @@ LError: /* REPLCONF